From 5cd6ac66206079d08b791e8a1bfa00df1071cc04 Mon Sep 17 00:00:00 2001 From: Arul Date: Sun, 2 Feb 2025 17:18:34 +0800 Subject: [PATCH 01/77] feat: v1 of AI-generated comments --- lib/cadet/assessments/assessments.ex | 11 +- .../controllers/generate_ai_comments.ex | 172 ++++++++++++++++++ lib/cadet_web/router.ex | 6 + 3 files changed, 187 insertions(+), 2 deletions(-) create mode 100644 lib/cadet_web/controllers/generate_ai_comments.ex diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 244ebc742..70307d85d 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -2289,8 +2289,8 @@ defmodule Cadet.Assessments do @spec get_answers_in_submission(integer() | String.t()) :: {:ok, {[Answer.t()], Assessment.t()}} | {:error, {:bad_request, String.t()}} - def get_answers_in_submission(id) when is_ecto_id(id) do - answer_query = + def get_answers_in_submission(id, question_id \\ nil) when is_ecto_id(id) do + base_query = Answer |> where(submission_id: ^id) |> join(:inner, [a], q in assoc(a, :question)) @@ -2312,6 +2312,13 @@ defmodule Cadet.Assessments do {s, student: {st, user: u}, team: {t, team_members: {tm, student: {tms, user: tmu}}}} ) + answer_query = + if is_nil(question_id) do + base_query + else + base_query |> where(question_id: ^question_id) + end + answers = answer_query |> Repo.all() diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex new file mode 100644 index 000000000..85ede4a1c --- /dev/null +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -0,0 +1,172 @@ +defmodule CadetWeb.AICodeAnalysisController do + use CadetWeb, :controller + use PhoenixSwagger + require HTTPoison + + alias Cadet.Assessments + + @openai_api_url "https://api.groq.com/openai/v1/chat/completions" + @model "llama3-8b-8192" + @api_key "x" + + @doc """ + Fetches the question details and answers based on submissionid and questionid and generates AI-generated comments. + """ + def generate_ai_comments(conn, %{"submissionid" => submission_id, "questionid" => question_id}) + when is_ecto_id(submission_id) do + case Assessments.get_answers_in_submission(submission_id, question_id) do + {:ok, {answers, _assessment}} -> + analyze_code(conn, answers) + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end + end + + defp transform_answers(answers) do + Enum.map(answers, fn answer -> + %{ + id: answer.id, + comments: answer.comments, + autograding_status: answer.autograding_status, + autograding_results: answer.autograding_results, + code: answer.answer["code"], + question_id: answer.question_id, + question_content: answer.question["content"] + } + end) + end + + defp analyze_code(conn, answers) do + # Convert each struct into a map and select only the required fields + answers_json = + answers + |> Enum.map(fn answer -> + question_data = + if answer.question do + %{ + id: answer.question_id, + content: Map.get(answer.question.question, "content") + } + else + %{ + id: nil, + content: nil + } + end + + answer + |> Map.from_struct() + |> Map.take([ + :id, + :comments, + :autograding_status, + :autograding_results, + :answer + ]) + |> Map.put(:question, question_data) + end) + |> Jason.encode!() + + prompt = """ + The code below was written in JavaScript. + + Analyze the following submitted answers and provide feedback on correctness, readability, efficiency, and improvements: + + Provide minimum 3 comment suggestions and maximum 5 comment suggestions. Keep each comment suggestion concise and specific, less than 100 words. + + Only provide your comment suggestions in the output and nothing else. + + Your output should be in the following format. + + DO NOT start the output with |||. Separate each suggestion using |||. + + DO NOT add spaces before or after the |||. + + Only provide the comment suggestions and separate each comment suggestion by using triple pipes ("|||"). + + For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." + + Do not provide any other information in the output, like "Here are the comment suggestions for the first answer" + + Do not include any bullet points, number lists, or any other formatting in your output. Just plain text comments, separated by triple pipes. + + #{answers_json} + """ + + body = + %{ + model: @model, + messages: [ + %{role: "system", content: "You are an expert software engineer and educator."}, + %{role: "user", content: prompt} + ], + temperature: 0.5 + } + |> Jason.encode!() + + headers = [ + {"Authorization", "Bearer #{@api_key}"}, + {"Content-Type", "application/json"} + ] + + case HTTPoison.post(@openai_api_url, body, headers) do + {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> + case Jason.decode(body) do + {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> + IO.inspect(response, label: "DEBUG: Raw AI Response") + comments_list = String.split(response, "|||") + + filtered_comments = + Enum.filter(comments_list, fn comment -> + String.trim(comment) != "" + end) + + json(conn, %{"comments" => filtered_comments}) + + {:error, _} -> + json(conn, %{"error" => "Failed to parse response from OpenAI API"}) + end + + {:ok, %HTTPoison.Response{status_code: status, body: body}} -> + json(conn, %{"error" => "API request failed with status #{status}: #{body}"}) + + {:error, %HTTPoison.Error{reason: reason}} -> + json(conn, %{"error" => "HTTP request error: #{inspect(reason)}"}) + end + end + + swagger_path :generate_ai_comments do + post("/courses/{courseId}/admin/generate-comments/{submissionId}/{questionId}") + + summary("Generate AI comments for a given submission.") + + security([%{JWT: []}]) + + consumes("application/json") + produces("application/json") + + parameters do + submissionId(:path, :integer, "submission id", required: true) + questionId(:path, :integer, "question id", required: true) + end + + response(200, "OK", Schema.ref(:GenerateAIComments)) + response(400, "Invalid or missing parameter(s) or submission and/or question not found") + response(401, "Unauthorized") + response(403, "Forbidden") + end + + def swagger_definitions do + %{ + GenerateAIComments: + swagger_schema do + properties do + comments(:string, "AI-generated comments on the submission answers") + end + end + } + end +end diff --git a/lib/cadet_web/router.ex b/lib/cadet_web/router.ex index 1ee304c01..3e8c1a155 100644 --- a/lib/cadet_web/router.ex +++ b/lib/cadet_web/router.ex @@ -201,6 +201,12 @@ defmodule CadetWeb.Router do post("/grading/:submissionid/autograde", AdminGradingController, :autograde_submission) post("/grading/:submissionid/:questionid", AdminGradingController, :update) + post( + "/generate-comments/:submissionid/:questionid", + AICodeAnalysisController, + :generate_ai_comments + ) + post( "/grading/:submissionid/:questionid/autograde", AdminGradingController, From 853ba847208e2689d27a4e42af37f0dbbe27e6d7 Mon Sep 17 00:00:00 2001 From: Arul Date: Thu, 13 Feb 2025 11:35:14 +0800 Subject: [PATCH 02/77] feat: added logging of inputs and outputs --- .gitignore | 3 + .../controllers/generate_ai_comments.ex | 116 ++++++++++-------- 2 files changed, 71 insertions(+), 48 deletions(-) diff --git a/.gitignore b/.gitignore index c6f53010c..80b3e3026 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,6 @@ erl_crash.dump # Generated lexer /src/source_lexer.erl + +# Ignore log files +/log \ No newline at end of file diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 85ede4a1c..ac669a4f3 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -2,27 +2,46 @@ defmodule CadetWeb.AICodeAnalysisController do use CadetWeb, :controller use PhoenixSwagger require HTTPoison + require Logger alias Cadet.Assessments - @openai_api_url "https://api.groq.com/openai/v1/chat/completions" - @model "llama3-8b-8192" - @api_key "x" + @openai_api_url "https://api.openai.com/v1/chat/completions" + @model "gpt-4o" + @api_key Application.get_env(:openai, :api_key) + + + # For logging outputs to a file + defp log_to_csv(submission_id, question_id, input, student_submission, output, error \\ nil) do + log_file = "log/ai_comments.csv" + File.mkdir_p!("log") + + timestamp = NaiveDateTime.utc_now() |> NaiveDateTime.to_string() + input_str = Jason.encode!(input) |> String.replace("\"", "\"\"") + student_submission_str = Jason.encode!(student_submission) |> String.replace("\"", "\"\"") + output_str = Jason.encode!(output) |> String.replace("\"", "\"\"") + error_str = if is_nil(error), do: "", else: Jason.encode!(error) |> String.replace("\"", "\"\"") + + csv_row = "\"#{timestamp}\",\"#{submission_id}\",\"#{question_id}\",\"#{input_str}\",\"#{student_submission_str}\",\"#{output_str}\",\"#{error_str}\"\n" + + File.write!(log_file, csv_row, [:append]) + end + @doc """ Fetches the question details and answers based on submissionid and questionid and generates AI-generated comments. """ def generate_ai_comments(conn, %{"submissionid" => submission_id, "questionid" => question_id}) - when is_ecto_id(submission_id) do - case Assessments.get_answers_in_submission(submission_id, question_id) do - {:ok, {answers, _assessment}} -> - analyze_code(conn, answers) - - {:error, {status, message}} -> - conn - |> put_status(status) - |> text(message) - end + when is_ecto_id(submission_id) do + case Assessments.get_answers_in_submission(submission_id, question_id) do + {:ok, {answers, _assessment}} -> + analyze_code(conn, answers, submission_id, question_id) + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end end defp transform_answers(answers) do @@ -39,8 +58,7 @@ defmodule CadetWeb.AICodeAnalysisController do end) end - defp analyze_code(conn, answers) do - # Convert each struct into a map and select only the required fields + defp analyze_code(conn, answers, submission_id, question_id) do answers_json = answers |> Enum.map(fn answer -> @@ -48,7 +66,8 @@ defmodule CadetWeb.AICodeAnalysisController do if answer.question do %{ id: answer.question_id, - content: Map.get(answer.question.question, "content") + content: Map.get(answer.question.question, "content"), + solution: Map.get(answer.question.question, "solution") } else %{ @@ -56,7 +75,6 @@ defmodule CadetWeb.AICodeAnalysisController do content: nil } end - answer |> Map.from_struct() |> Map.take([ @@ -64,76 +82,78 @@ defmodule CadetWeb.AICodeAnalysisController do :comments, :autograding_status, :autograding_results, - :answer + :answer, ]) |> Map.put(:question, question_data) end) |> Jason.encode!() - prompt = """ - The code below was written in JavaScript. + raw_prompt = """ + The code below was written in JavaScript. - Analyze the following submitted answers and provide feedback on correctness, readability, efficiency, and improvements: + Analyze the following submitted answers and provide feedback on correctness, readability, efficiency, and improvements: - Provide minimum 3 comment suggestions and maximum 5 comment suggestions. Keep each comment suggestion concise and specific, less than 100 words. + Provide minimum 3 comment suggestions and maximum 5 comment suggestions. Keep each comment suggestion concise and specific, less than 200 words. - Only provide your comment suggestions in the output and nothing else. + Only provide your comment suggestions in the output and nothing else. - Your output should be in the following format. + Your output should be in the following format. - DO NOT start the output with |||. Separate each suggestion using |||. + DO NOT start the output with |||. Separate each suggestion using |||. - DO NOT add spaces before or after the |||. + DO NOT add spaces before or after the |||. - Only provide the comment suggestions and separate each comment suggestion by using triple pipes ("|||"). + Only provide the comment suggestions and separate each comment suggestion by using triple pipes ("|||"). - For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." + For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." - Do not provide any other information in the output, like "Here are the comment suggestions for the first answer" + Do not provide any other information in the output, like "Here are the comment suggestions for the first answer" - Do not include any bullet points, number lists, or any other formatting in your output. Just plain text comments, separated by triple pipes. + Do not include any bullet points, number lists, or any other formatting in your output. Just plain text comments, separated by triple pipes ("|||"). + """ - #{answers_json} - """ + prompt = raw_prompt <> "\n" <> answers_json - body = - %{ - model: @model, - messages: [ - %{role: "system", content: "You are an expert software engineer and educator."}, - %{role: "user", content: prompt} - ], - temperature: 0.5 - } - |> Jason.encode!() + + input = %{ + model: @model, + messages: [ + %{role: "system", content: "You are an expert software engineer and educator."}, + %{role: "user", content: prompt} + ], + temperature: 0.5 + } |> Jason.encode!() headers = [ {"Authorization", "Bearer #{@api_key}"}, {"Content-Type", "application/json"} ] - case HTTPoison.post(@openai_api_url, body, headers) do + + case HTTPoison.post(@openai_api_url, input, headers) do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> case Jason.decode(body) do {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> - IO.inspect(response, label: "DEBUG: Raw AI Response") + log_to_csv(submission_id, question_id, raw_prompt, answers_json, response) comments_list = String.split(response, "|||") - filtered_comments = - Enum.filter(comments_list, fn comment -> - String.trim(comment) != "" - end) + filtered_comments = Enum.filter(comments_list, fn comment -> + String.trim(comment) != "" + end) json(conn, %{"comments" => filtered_comments}) {:error, _} -> + log_to_csv(submission_id, question_id, raw_prompt, answers_json, nil, "Failed to parse response from OpenAI API") json(conn, %{"error" => "Failed to parse response from OpenAI API"}) end {:ok, %HTTPoison.Response{status_code: status, body: body}} -> + log_to_csv(submission_id, question_id, raw_prompt, answers_json, nil, "API request failed with status #{status}") json(conn, %{"error" => "API request failed with status #{status}: #{body}"}) {:error, %HTTPoison.Error{reason: reason}} -> + log_to_csv(submission_id, question_id, raw_prompt, answers_json, nil, reason) json(conn, %{"error" => "HTTP request error: #{inspect(reason)}"}) end end From 4c37d14ed886177aea10318238eca186314c1afe Mon Sep 17 00:00:00 2001 From: Arul Date: Fri, 21 Feb 2025 10:17:04 +0800 Subject: [PATCH 03/77] Update generate_ai_comments.ex --- .../controllers/generate_ai_comments.ex | 56 ++++++++++++------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index ac669a4f3..8f16ad6d2 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -89,27 +89,45 @@ defmodule CadetWeb.AICodeAnalysisController do |> Jason.encode!() raw_prompt = """ - The code below was written in JavaScript. - - Analyze the following submitted answers and provide feedback on correctness, readability, efficiency, and improvements: - - Provide minimum 3 comment suggestions and maximum 5 comment suggestions. Keep each comment suggestion concise and specific, less than 200 words. - - Only provide your comment suggestions in the output and nothing else. - - Your output should be in the following format. - - DO NOT start the output with |||. Separate each suggestion using |||. - - DO NOT add spaces before or after the |||. - - Only provide the comment suggestions and separate each comment suggestion by using triple pipes ("|||"). + The code below is written in Source, a variant of JavaScript that comes with a rich set of built-in constants and functions. Below is a summary of some key built-in entities available in Source: + + Constants: + - Infinity: The special number value representing infinity. + - NaN: The special number value for "not a number." + - undefined: The special value for an undefined variable. + - math_PI: The constant π (approximately 3.14159). + - math_E: Euler's number (approximately 2.71828). + + Functions: + - __access_export__(exports, lookup_name): Searches for a name in an exports data structure. + - accumulate(f, initial, xs): Reduces a list by applying a binary function from right-to-left. + - append(xs, ys): Appends list ys to the end of list xs. + - char_at(s, i): Returns the character at index i of string s. + - display(v, s): Displays value v (optionally preceded by string s) in the console. + - filter(pred, xs): Returns a new list with elements of xs that satisfy the predicate pred. + - for_each(f, xs): Applies function f to each element of the list xs. + - get_time(): Returns the current time in milliseconds. + - is_list(xs): Checks whether xs is a proper list. + - length(xs): Returns the number of elements in list xs. + - list(...): Constructs a list from the provided values. + - map(f, xs): Applies function f to each element of list xs. + - math_abs(x): Returns the absolute value of x. + - math_ceil(x): Rounds x up to the nearest integer. + - math_floor(x): Rounds x down to the nearest integer. + - pair(x, y): A primitive function that makes a pair whose head (first component) is x and whose tail (second component) is y. + - head(xs): Returns the first element of pair xs. + - tail(xs): Returns the second element of pair xs. + - math_random(): Returns a random number between 0 (inclusive) and 1 (exclusive). + + (For a full list of built-in functions and constants, refer to the Source documentation.) + + Analyze the following submitted answers and provide detailed feedback on correctness, readability, efficiency, and possible improvements. Your evaluation should consider both standard JavaScript features and the additional built-in functions unique to Source. + + Provide between 3 and 5 concise comment suggestions, each under 200 words. + + Your output must include only the comment suggestions, separated exclusively by triple pipes ("|||") with no spaces before or after the pipes, and without any additional formatting, bullet points, or extra text. For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." - - Do not provide any other information in the output, like "Here are the comment suggestions for the first answer" - - Do not include any bullet points, number lists, or any other formatting in your output. Just plain text comments, separated by triple pipes ("|||"). """ prompt = raw_prompt <> "\n" <> answers_json From 8192b3de4d06fdf6e569c5831ddc085eb9229124 Mon Sep 17 00:00:00 2001 From: Arul Date: Mon, 17 Mar 2025 17:35:35 +0800 Subject: [PATCH 04/77] feat: function to save outputs to database --- lib/cadet/ai_comments.ex | 50 +++++++ lib/cadet/ai_comments/ai_comment.ex | 23 ++++ .../controllers/generate_ai_comments.ex | 103 ++++++++++++--- .../20250220103623_create_ai_comments.exs | 20 +++ .../ai_code_analysis_controller_test.exs | 124 ++++++++++++++++++ 5 files changed, 304 insertions(+), 16 deletions(-) create mode 100644 lib/cadet/ai_comments.ex create mode 100644 lib/cadet/ai_comments/ai_comment.ex create mode 100644 priv/repo/migrations/20250220103623_create_ai_comments.exs create mode 100644 test/cadet_web/controllers/ai_code_analysis_controller_test.exs diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex new file mode 100644 index 000000000..3f42c9004 --- /dev/null +++ b/lib/cadet/ai_comments.ex @@ -0,0 +1,50 @@ +defmodule Cadet.AIComments do + import Ecto.Query + alias Cadet.Repo + alias Cadet.AIComments.AIComment + + @doc """ + Creates a new AI comment log entry. + """ + def create_ai_comment(attrs \\ %{}) do + %AIComment{} + |> AIComment.changeset(attrs) + |> Repo.insert() + end + + @doc """ + Gets an AI comment by ID. + """ + def get_ai_comment!(id), do: Repo.get!(AIComment, id) + + @doc """ + Gets AI comments for a specific submission and question. + """ + def get_ai_comments_for_submission(submission_id, question_id) do + from(c in AIComment, + where: c.submission_id == ^submission_id and c.question_id == ^question_id, + order_by: [desc: c.inserted_at] + ) + |> Repo.all() + end + + @doc """ + Updates the final comment for a specific submission and question. + Returns the most recent comment entry for that submission/question. + """ + def update_final_comment(submission_id, question_id, final_comment) do + from(c in AIComment, + where: c.submission_id == ^submission_id and c.question_id == ^question_id, + order_by: [desc: c.inserted_at], + limit: 1 + ) + |> Repo.one() + |> case do + nil -> {:error, :not_found} + comment -> + comment + |> AIComment.changeset(%{final_comment: final_comment}) + |> Repo.update() + end + end +end diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex new file mode 100644 index 000000000..575d1cc10 --- /dev/null +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -0,0 +1,23 @@ +defmodule Cadet.AIComments.AIComment do + use Ecto.Schema + import Ecto.Changeset + + schema "ai_comment_logs" do + field :submission_id, :integer + field :question_id, :integer + field :raw_prompt, :string + field :answers_json, :string + field :response, :string + field :error, :string + field :comment_chosen, :string + field :final_comment, :string + + timestamps() + end + + def changeset(ai_comment, attrs) do + ai_comment + |> cast(attrs, [:submission_id, :question_id, :raw_prompt, :answers_json, :response, :error, :comment_chosen, :final_comment]) + |> validate_required([:submission_id, :question_id, :raw_prompt, :answers_json]) + end +end diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 8f16ad6d2..a543d5518 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -5,28 +5,51 @@ defmodule CadetWeb.AICodeAnalysisController do require Logger alias Cadet.Assessments + alias Cadet.AIComments @openai_api_url "https://api.openai.com/v1/chat/completions" @model "gpt-4o" @api_key Application.get_env(:openai, :api_key) + # For logging outputs to both database and file + defp log_comment(submission_id, question_id, raw_prompt, answers_json, response, error \\ nil) do + # Log to database + attrs = %{ + submission_id: submission_id, + question_id: question_id, + raw_prompt: raw_prompt, + answers_json: answers_json, + response: response, + error: error, + inserted_at: NaiveDateTime.utc_now() + } - # For logging outputs to a file - defp log_to_csv(submission_id, question_id, input, student_submission, output, error \\ nil) do - log_file = "log/ai_comments.csv" - File.mkdir_p!("log") + case AIComments.create_ai_comment(attrs) do + {:ok, comment} -> {:ok, comment} + {:error, changeset} -> + Logger.error("Failed to log AI comment to database: #{inspect(changeset.errors)}") + {:error, changeset} + end - timestamp = NaiveDateTime.utc_now() |> NaiveDateTime.to_string() - input_str = Jason.encode!(input) |> String.replace("\"", "\"\"") - student_submission_str = Jason.encode!(student_submission) |> String.replace("\"", "\"\"") - output_str = Jason.encode!(output) |> String.replace("\"", "\"\"") - error_str = if is_nil(error), do: "", else: Jason.encode!(error) |> String.replace("\"", "\"\"") + # Log to file + try do + log_file = "log/ai_comments.csv" + File.mkdir_p!("log") - csv_row = "\"#{timestamp}\",\"#{submission_id}\",\"#{question_id}\",\"#{input_str}\",\"#{student_submission_str}\",\"#{output_str}\",\"#{error_str}\"\n" + timestamp = NaiveDateTime.utc_now() |> NaiveDateTime.to_string() + raw_prompt_str = Jason.encode!(raw_prompt) |> String.replace("\"", "\"\"") + answers_json_str = answers_json |> String.replace("\"", "\"\"") + response_str = if is_nil(response), do: "", else: response |> String.replace("\"", "\"\"") + error_str = if is_nil(error), do: "", else: error |> String.replace("\"", "\"\"") - File.write!(log_file, csv_row, [:append]) - end + csv_row = "\"#{timestamp}\",\"#{submission_id}\",\"#{question_id}\",\"#{raw_prompt_str}\",\"#{answers_json_str}\",\"#{response_str}\",\"#{error_str}\"\n" + File.write!(log_file, csv_row, [:append]) + rescue + e -> + Logger.error("Failed to log AI comment to file: #{inspect(e)}") + end + end @doc """ Fetches the question details and answers based on submissionid and questionid and generates AI-generated comments. @@ -127,7 +150,13 @@ defmodule CadetWeb.AICodeAnalysisController do Your output must include only the comment suggestions, separated exclusively by triple pipes ("|||") with no spaces before or after the pipes, and without any additional formatting, bullet points, or extra text. + Comments and documentation in the code are not necessary for the code, do not penalise based on that, do not suggest to add comments as well. + + Follow the XP scoring guideline provided below in the question prompt, do not be too harsh! + For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." + + #Agent Role# You are a kind coding assistant and mentor. #General Instruction on comment style# There is a programming question, and you have to write a comment on the student's answer to the programming question. Note that your reply is addressed directly to the student, so prevent any sentence out of the desired comment in your response to this prompt. The comment includes feedback on the solution's correctness. Suggest improvement areas if necessary. If the answer is incorrect, declare why the answer is wrong, but only give general hints as suggestions and avoid explaining the Right solution. You should keep your tone friendly even if the answer is incorrect and you want to suggest improvements. If there are several problems in the solution, you have to mention all of them. The maximum length of your reply to this prompt can be 50 words. If the answer is correct and you don't have any suggestions, only write "Great job!". #Prequistic knowledge to solve the question# In this question, you're going to work with Runes. Predefined Runes include heart, circle, square, sail, rcross, nova, corner, and blank. You can access these Runes using their names. You can only use predeclared functions, including "show," "beside," "stack," "beside_frac," "stack_frack," "make_cross," "quarter_turn_left," "quarter_turn_right," "turn_upside_down." These functions are defined below: 1. [Function "show" renders the specified Rune in a tab as a basic drawing. Function prototype: show(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns the specified Rune. Example: "show(heart)" shows a heart shape rune.] 2. [Function "beside" makes a new Rune from two given Runes by placing the first on the left of the second, both occupying equal portions of the width of the result. Function prototype: beside(rune1: Rune, rune2: Rune): Rune Prototype Description: It takes two parameters of type Rune, rune1 and rune2, as input and returns a Rune. Example 1: "beside(r1, r2)", places r1 on the left of the r2. Example 2: "beside(stack(r1, r2), stack(r3, r4))" places the output of stack(r1, r2) on the left of output of stack(r3, r4). ] 3. [Function "stack" makes a new Rune from two given Runes by placing the first one on top of the second one, each occupying equal parts of the height of the result. Function prototype: stack(rune1: Rune, rune2: Rune): Rune Prototype Description: It takes two parameters of type Rune, rune1 and rune2, as input and returns a Rune. Example1: "stack(r1, r2)" places r1 on top of r2. Example 2: "Stack(beside(r1, r2), beside(r3, r4))" places output of beside(r1, r2) on top of the output of beside(r3, r4).] 4. [Function "beside_frack" Makes a new Rune from two given Runes by placing the first on the left of the second such that the first one occupies a frac portion of the width of the result and the second the rest. Function Prototype: beside_frac(frac: number, rune1: Rune, rune2: Rune): Rune Prototype Description: It takes a number between 0 and 1 as "frac" and two parameters of type Rune, "rune1" and "rune2," as input and returns a Rune parameter. Example 1: "beside_frac(1/2, heart, circle) places a heart on the left of the circle, and both occupy 1/2 of the plane." Example 2: "beside_frac(1/4, heart, circle) places a heart on the left of the circle. The heart occupies 1/4 of the plane, and the circle occupies 3/4 of the plane."] 5. [Function "stack_frack" Makes a new Rune from two given Runes by placing the first on top of the second such that the first one occupies a frac portion of the height of the result and the second the rest. Function Prototype:stack_frac(frac: number, rune1: Rune, rune2: Rune): Rune Prototype Description: It takes a number between 0 and 1 as "frac" and two parameters of type Rune, "rune1" and "rune2," as input and returns a Rune parameter. Example 1: "stack_frac(1/2, heart, circle) places a heart on top of the circle, and both occupy 1/2 of the plane." Example 2: "stack_frac(1/4, heart, circle) places a heart on top of the circle. The heart occupies 1/4 of the plane, and the circle occupies 3/4 of the plane."] 6. [Function "make_cross" makes a new Rune from a given Rune by arranging it into a square for copies of the given Rune in different orientations. Function Prototype: make_cross(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example: "make_cross(heart)" places a heart shape rune on the bottom-left, a 90-degree clockwise rotated heart on the top-left, a 180-degree clockwise rotated heart on the top-right, and a 270-degree clockwise rotated heart on the bottom-right. The final Rune consists of four runes.] 7. [Function "quarter_turn_left" Makes a new Rune from a given Rune by turning it a quarter-turn in an anti-clockwise direction. Function prototype: quarter_turn_right(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example 1: "quarter_turn_left(heart)" rotates the heart shape rune 90 degrees in an anti-clockwise direction. Example 2: "quarter_turn_left(stack(r1, r2))" rotates the output of stack(r1, r2) 90 degrees in an anti-clockwise direction. ] 8. [Function "quarter_turn_right" makes a new Rune from a given Rune by turning it a quarter-turn around the center in a clockwise direction. Function prototype: quarter_turn_right(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example 1: "quarter_turn_right(heart)" rotates the heart shape rune 90 degrees in a clockwise direction. Example 2: "quarter_turn_right(stack(r1, r2))" rotates the output of stack(r1, r2) 90 degrees in a clockwise direction. ] 9. [Function "turn_upside_down" makes a new Rune from a given Rune by turning it upside-down. Function prototype: turn_upside_down(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example 1: "turn_upside_down(heart)" rotates a heart shape rune 180 degrees in a clockwise direction. Example 2: "turn_upside_down(stack(r1, r2))" rotates the output of stack(r1, r2) 180 degrees in a clockwise direction.] You must only use the Runes and functions declared above and avoid importing any module in your program. You can pass the output of each function as input to another function. For example, consider beside(stack(r2, r1), stack(r3, r4)). First, the inner stack functions get executed. r2 goes to the left of r1, and r3 goes to the left of r4. Then the output Rune of each stack works as input of beside function. meaning output of stak(r2, r1) goes on top of output of stack(r3,r4). Avoid hard coding. #Programming question# Write a function hook that takes a fraction "frac" as an input and creates a 'hook' pattern. The fraction input determines the size of the base of the hook. The output rune: [Imagine a rectangle divided into two horizontal sections. Each section is the height of a square. Top Section: This section is simply a filled square. Bottom Section: The bottom section is also the size of a square. However, it's divided into two equal parts vertically. The left side of this square is filled (so it looks like a rectangle that's half the width of the square). The right side of this square is blank or empty. So, if you place these two sections on top of one another, you get: A full square on top. Directly below it, on the left side, you have a half-filled square (a rectangle), and on the right side, it's empty. This forms a "hook" rune, with the hook part facing to the left. The overall rune is a square with two times the height of the original squares used to create it. Examples: hook(1): It's simply a square rune. hook(0): A filled square at the top. An empty or blank space at the bottom of the same size as the square. hook(1/2): A full square on top. Below that, on the right side, there's another filled square that's half the width of the full square. On the left side, it's empty. hook(1/5): A full square on top. Below that, on the right side, there's a very thin filled rectangle (only 1/5 the width of the square). The rest (4/5) to the right is empty.] You will only need to use the square and blank primitive runes and transform them to get the hook. Implement your function in the code below: "function hook(frac) { // your answer here } // Test show(hook(1/5));" #Sample Solution and feedback# 1. "function hook(frac) { return stack(square, quarter_turn_right( stack_frac(frac, square, blank))); } // Test show(hook(1/5));" - Great job! 2. "function hook(frac) { return frac === 1 ? square : frac === 0 ? stack(square,blank) : stack(square,beside_frac(1-frac, blank, square)); } show(hook(1/5));" - Excellent work! 3."function hook(frac) { return stack(square, quarter_turn_left( stack_frac(1-frac, blank, square))); } show(hook(1/5));" -Great job! 4."function hook(frac) { // your answer here return stack_frac(1/2,square, beside_frac(1-frac,blank,square)); } // Test show(hook(1/5));" -Good job, However stack_frac(1 / 2, etc) could have been simplified by merely using stack. """ prompt = raw_prompt <> "\n" <> answers_json @@ -152,7 +181,7 @@ defmodule CadetWeb.AICodeAnalysisController do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> case Jason.decode(body) do {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> - log_to_csv(submission_id, question_id, raw_prompt, answers_json, response) + log_comment(submission_id, question_id, raw_prompt, answers_json, response) comments_list = String.split(response, "|||") filtered_comments = Enum.filter(comments_list, fn comment -> @@ -162,20 +191,34 @@ defmodule CadetWeb.AICodeAnalysisController do json(conn, %{"comments" => filtered_comments}) {:error, _} -> - log_to_csv(submission_id, question_id, raw_prompt, answers_json, nil, "Failed to parse response from OpenAI API") + log_comment(submission_id, question_id, raw_prompt, answers_json, nil, "Failed to parse response from OpenAI API") json(conn, %{"error" => "Failed to parse response from OpenAI API"}) end {:ok, %HTTPoison.Response{status_code: status, body: body}} -> - log_to_csv(submission_id, question_id, raw_prompt, answers_json, nil, "API request failed with status #{status}") + log_comment(submission_id, question_id, raw_prompt, answers_json, nil, "API request failed with status #{status}") json(conn, %{"error" => "API request failed with status #{status}: #{body}"}) {:error, %HTTPoison.Error{reason: reason}} -> - log_to_csv(submission_id, question_id, raw_prompt, answers_json, nil, reason) + log_comment(submission_id, question_id, raw_prompt, answers_json, nil, reason) json(conn, %{"error" => "HTTP request error: #{inspect(reason)}"}) end end + @doc """ + Saves the final comment chosen for a submission. + """ + def save_final_comment(conn, %{"submissionid" => submission_id, "questionid" => question_id, "comment" => comment}) do + case AIComments.update_final_comment(submission_id, question_id, comment) do + {:ok, _updated_comment} -> + json(conn, %{"status" => "success"}) + {:error, changeset} -> + conn + |> put_status(:unprocessable_entity) + |> json(%{"error" => "Failed to save final comment"}) + end + end + swagger_path :generate_ai_comments do post("/courses/{courseId}/admin/generate-comments/{submissionId}/{questionId}") @@ -197,6 +240,28 @@ defmodule CadetWeb.AICodeAnalysisController do response(403, "Forbidden") end + swagger_path :save_final_comment do + post("/courses/{courseId}/admin/save-final-comment/{submissionId}/{questionId}") + + summary("Save the final comment chosen for a submission.") + + security([%{JWT: []}]) + + consumes("application/json") + produces("application/json") + + parameters do + submissionId(:path, :integer, "submission id", required: true) + questionId(:path, :integer, "question id", required: true) + comment(:body, :string, "The final comment to save", required: true) + end + + response(200, "OK", Schema.ref(:SaveFinalComment)) + response(400, "Invalid or missing parameter(s)") + response(401, "Unauthorized") + response(403, "Forbidden") + end + def swagger_definitions do %{ GenerateAIComments: @@ -204,6 +269,12 @@ defmodule CadetWeb.AICodeAnalysisController do properties do comments(:string, "AI-generated comments on the submission answers") end + end, + SaveFinalComment: + swagger_schema do + properties do + status(:string, "Status of the operation") + end end } end diff --git a/priv/repo/migrations/20250220103623_create_ai_comments.exs b/priv/repo/migrations/20250220103623_create_ai_comments.exs new file mode 100644 index 000000000..6338dccd8 --- /dev/null +++ b/priv/repo/migrations/20250220103623_create_ai_comments.exs @@ -0,0 +1,20 @@ +defmodule Cadet.Repo.Migrations.CreateAiCommentLogs do + use Ecto.Migration + + def change do + create table(:ai_comment_logs) do + add(:submission_id, :integer, null: false) + add(:question_id, :integer, null: false) + add(:raw_prompt, :text, null: false) + add(:answers_json, :text, null: false) + add(:response, :text) + add(:error, :text) + add(:comment_chosen, :text) + add(:final_comment, :text) + timestamps() + end + + create(index(:ai_comment_logs, [:submission_id])) + create(index(:ai_comment_logs, [:question_id])) + end +end diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs new file mode 100644 index 000000000..559c4ea9e --- /dev/null +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -0,0 +1,124 @@ +defmodule CadetWeb.AICodeAnalysisControllerTest do + use CadetWeb.ConnCase + alias Cadet.AIComments + alias Cadet.AIComments.AIComment + + setup do + # Clean up test files before each test + log_file = "log/ai_comments.csv" + File.rm(log_file) + :ok + end + + describe "generate_ai_comments" do + test "successfully logs comments to both database and file", %{conn: conn} do + # Test data + submission_id = 123 + question_id = 456 + raw_prompt = "Test prompt" + answers_json = ~s({"test": "data"}) + mock_response = "Comment 1|||Comment 2|||Comment 3" + + # Make the API call + response = + conn + |> post(Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id)) + |> json_response(200) + + # Verify database entry + comments = Repo.all(AIComment) + assert length(comments) > 0 + latest_comment = List.first(comments) + assert latest_comment.submission_id == submission_id + assert latest_comment.question_id == question_id + assert latest_comment.raw_prompt != nil + assert latest_comment.answers_json != nil + + # Verify CSV file + log_file = "log/ai_comments.csv" + assert File.exists?(log_file) + file_content = File.read!(log_file) + + # Check if CSV contains the required data + assert file_content =~ Integer.to_string(submission_id) + assert file_content =~ Integer.to_string(question_id) + end + + test "logs error when API call fails", %{conn: conn} do + # Test data with invalid submission_id to trigger error + submission_id = -1 + question_id = 456 + + # Make the API call that should fail + response = + conn + |> post(Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id)) + |> json_response(400) + + # Verify error is logged in database + comments = Repo.all(AIComment) + assert length(comments) > 0 + error_log = List.first(comments) + assert error_log.error != nil + assert error_log.submission_id == submission_id + assert error_log.question_id == question_id + + # Verify error is logged in CSV + log_file = "log/ai_comments.csv" + assert File.exists?(log_file) + file_content = File.read!(log_file) + assert file_content =~ Integer.to_string(submission_id) + assert file_content =~ Integer.to_string(question_id) + assert file_content =~ "error" + end + end + + describe "save_final_comment" do + test "successfully saves final comment", %{conn: conn} do + # First create a comment entry + submission_id = 123 + question_id = 456 + raw_prompt = "Test prompt" + answers_json = ~s({"test": "data"}) + response = "Comment 1|||Comment 2|||Comment 3" + + {:ok, _comment} = AIComments.create_ai_comment(%{ + submission_id: submission_id, + question_id: question_id, + raw_prompt: raw_prompt, + answers_json: answers_json, + response: response + }) + + # Now save the final comment + final_comment = "This is the chosen final comment" + response = + conn + |> post(Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), %{ + comment: final_comment + }) + |> json_response(200) + + assert response["status"] == "success" + + # Verify the comment was saved + comment = Repo.get_by(AIComment, submission_id: submission_id, question_id: question_id) + assert comment.final_comment == final_comment + end + + test "returns error when no comment exists", %{conn: conn} do + submission_id = 999 + question_id = 888 + final_comment = "This comment should not be saved" + + response = + conn + |> post(Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), %{ + comment: final_comment + }) + |> json_response(422) + + assert response["error"] == "Failed to save final comment" + end + end +end From 8a235b3251c5c80e01a3a738647e686dd055f360 Mon Sep 17 00:00:00 2001 From: Eugene Oh Yun Zheng Date: Tue, 18 Mar 2025 10:56:56 +0800 Subject: [PATCH 05/77] Format answers json before sending to LLM --- .../controllers/generate_ai_comments.ex | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index a543d5518..4557d52ed 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -81,6 +81,37 @@ defmodule CadetWeb.AICodeAnalysisController do end) end + def format_answers(json_string) do + {:ok, answers} = Jason.decode(json_string) + + answers + |> Enum.map(&format_answer/1) + |> Enum.join("\n\n") + end + + defp format_answer(answer) do + """ + **Question ID: #{answer["question"]["id"]}** + + **Question:** + #{answer["question"]["content"]} + + **Solution:** + ``` + #{answer["question"]["solution"]} + ``` + + **Answer:** + ``` + #{answer["answer"]["code"]} + ``` + + **Autograding Status:** #{answer["autograding_status"]} + **Autograding Results:** #{answer["autograding_results"]} + **Comments:** #{answer["comments"] || "None"} + """ + end + defp analyze_code(conn, answers, submission_id, question_id) do answers_json = answers @@ -110,6 +141,7 @@ defmodule CadetWeb.AICodeAnalysisController do |> Map.put(:question, question_data) end) |> Jason.encode!() + |> format_answers() raw_prompt = """ The code below is written in Source, a variant of JavaScript that comes with a rich set of built-in constants and functions. Below is a summary of some key built-in entities available in Source: From d384e06c9e13b451d045f53e6b3aadf308896694 Mon Sep 17 00:00:00 2001 From: Eugene Oh Yun Zheng Date: Tue, 18 Mar 2025 11:11:12 +0800 Subject: [PATCH 06/77] Add LLM Prompt to question params when submitting assessment xml file --- lib/cadet/assessments/question_types/programming_question.ex | 3 ++- lib/cadet/jobs/xml_parser.ex | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/cadet/assessments/question_types/programming_question.ex b/lib/cadet/assessments/question_types/programming_question.ex index a39625de3..245e8ef11 100644 --- a/lib/cadet/assessments/question_types/programming_question.ex +++ b/lib/cadet/assessments/question_types/programming_question.ex @@ -13,13 +13,14 @@ defmodule Cadet.Assessments.QuestionTypes.ProgrammingQuestion do field(:template, :string) field(:postpend, :string, default: "") field(:solution, :string) + field(:llm_prompt, :string) embeds_many(:public, Testcase) embeds_many(:opaque, Testcase) embeds_many(:secret, Testcase) end @required_fields ~w(content template)a - @optional_fields ~w(solution prepend postpend)a + @optional_fields ~w(solution prepend postpend llm_prompt)a def changeset(question, params \\ %{}) do question diff --git a/lib/cadet/jobs/xml_parser.ex b/lib/cadet/jobs/xml_parser.ex index b49128506..abf16c8cd 100644 --- a/lib/cadet/jobs/xml_parser.ex +++ b/lib/cadet/jobs/xml_parser.ex @@ -202,7 +202,8 @@ defmodule Cadet.Updater.XMLParser do prepend: ~x"./SNIPPET/PREPEND/text()" |> transform_by(&process_charlist/1), template: ~x"./SNIPPET/TEMPLATE/text()" |> transform_by(&process_charlist/1), postpend: ~x"./SNIPPET/POSTPEND/text()" |> transform_by(&process_charlist/1), - solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1) + solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1), + llm_prompt: ~x"./LLM_GRADING_PROMPT/text()" |> transform_by(&process_charlist/1) ), entity |> xmap( From 98feac2402403a71f6061ab811fe0a472eadb0cb Mon Sep 17 00:00:00 2001 From: Eugene Oh Yun Zheng Date: Tue, 18 Mar 2025 12:59:16 +0800 Subject: [PATCH 07/77] Add LLM Prompt to api response when grading view is open --- lib/cadet_web/helpers/assessments_helpers.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cadet_web/helpers/assessments_helpers.ex b/lib/cadet_web/helpers/assessments_helpers.ex index 967df1131..7e03e5d83 100644 --- a/lib/cadet_web/helpers/assessments_helpers.ex +++ b/lib/cadet_web/helpers/assessments_helpers.ex @@ -177,6 +177,7 @@ defmodule CadetWeb.AssessmentsHelpers do prepend: "prepend", solutionTemplate: "template", postpend: "postpend", + llm_prompt: "llm_prompt", testcases: build_testcases(all_testcases?) }) From 7716d57732e71d678f227b6de07be69d4a5b1cc7 Mon Sep 17 00:00:00 2001 From: Arul Date: Wed, 19 Mar 2025 14:25:01 +0800 Subject: [PATCH 08/77] feat: added llm_prompt from qn to raw_prompt --- .../controllers/generate_ai_comments.ex | 36 ++++++++----- test/support/ai_comments_test_helpers.ex | 50 +++++++++++++++++++ 2 files changed, 74 insertions(+), 12 deletions(-) create mode 100644 test/support/ai_comments_test_helpers.ex diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 4557d52ed..d96109da6 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -121,12 +121,14 @@ defmodule CadetWeb.AICodeAnalysisController do %{ id: answer.question_id, content: Map.get(answer.question.question, "content"), - solution: Map.get(answer.question.question, "solution") + solution: Map.get(answer.question.question, "solution"), + llm_prompt: Map.get(answer.question.question, "llm_prompt") } else %{ id: nil, - content: nil + content: nil, + llm_prompt: nil } end answer @@ -187,12 +189,22 @@ defmodule CadetWeb.AICodeAnalysisController do Follow the XP scoring guideline provided below in the question prompt, do not be too harsh! For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." - - #Agent Role# You are a kind coding assistant and mentor. #General Instruction on comment style# There is a programming question, and you have to write a comment on the student's answer to the programming question. Note that your reply is addressed directly to the student, so prevent any sentence out of the desired comment in your response to this prompt. The comment includes feedback on the solution's correctness. Suggest improvement areas if necessary. If the answer is incorrect, declare why the answer is wrong, but only give general hints as suggestions and avoid explaining the Right solution. You should keep your tone friendly even if the answer is incorrect and you want to suggest improvements. If there are several problems in the solution, you have to mention all of them. The maximum length of your reply to this prompt can be 50 words. If the answer is correct and you don't have any suggestions, only write "Great job!". #Prequistic knowledge to solve the question# In this question, you're going to work with Runes. Predefined Runes include heart, circle, square, sail, rcross, nova, corner, and blank. You can access these Runes using their names. You can only use predeclared functions, including "show," "beside," "stack," "beside_frac," "stack_frack," "make_cross," "quarter_turn_left," "quarter_turn_right," "turn_upside_down." These functions are defined below: 1. [Function "show" renders the specified Rune in a tab as a basic drawing. Function prototype: show(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns the specified Rune. Example: "show(heart)" shows a heart shape rune.] 2. [Function "beside" makes a new Rune from two given Runes by placing the first on the left of the second, both occupying equal portions of the width of the result. Function prototype: beside(rune1: Rune, rune2: Rune): Rune Prototype Description: It takes two parameters of type Rune, rune1 and rune2, as input and returns a Rune. Example 1: "beside(r1, r2)", places r1 on the left of the r2. Example 2: "beside(stack(r1, r2), stack(r3, r4))" places the output of stack(r1, r2) on the left of output of stack(r3, r4). ] 3. [Function "stack" makes a new Rune from two given Runes by placing the first one on top of the second one, each occupying equal parts of the height of the result. Function prototype: stack(rune1: Rune, rune2: Rune): Rune Prototype Description: It takes two parameters of type Rune, rune1 and rune2, as input and returns a Rune. Example1: "stack(r1, r2)" places r1 on top of r2. Example 2: "Stack(beside(r1, r2), beside(r3, r4))" places output of beside(r1, r2) on top of the output of beside(r3, r4).] 4. [Function "beside_frack" Makes a new Rune from two given Runes by placing the first on the left of the second such that the first one occupies a frac portion of the width of the result and the second the rest. Function Prototype: beside_frac(frac: number, rune1: Rune, rune2: Rune): Rune Prototype Description: It takes a number between 0 and 1 as "frac" and two parameters of type Rune, "rune1" and "rune2," as input and returns a Rune parameter. Example 1: "beside_frac(1/2, heart, circle) places a heart on the left of the circle, and both occupy 1/2 of the plane." Example 2: "beside_frac(1/4, heart, circle) places a heart on the left of the circle. The heart occupies 1/4 of the plane, and the circle occupies 3/4 of the plane."] 5. [Function "stack_frack" Makes a new Rune from two given Runes by placing the first on top of the second such that the first one occupies a frac portion of the height of the result and the second the rest. Function Prototype:stack_frac(frac: number, rune1: Rune, rune2: Rune): Rune Prototype Description: It takes a number between 0 and 1 as "frac" and two parameters of type Rune, "rune1" and "rune2," as input and returns a Rune parameter. Example 1: "stack_frac(1/2, heart, circle) places a heart on top of the circle, and both occupy 1/2 of the plane." Example 2: "stack_frac(1/4, heart, circle) places a heart on top of the circle. The heart occupies 1/4 of the plane, and the circle occupies 3/4 of the plane."] 6. [Function "make_cross" makes a new Rune from a given Rune by arranging it into a square for copies of the given Rune in different orientations. Function Prototype: make_cross(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example: "make_cross(heart)" places a heart shape rune on the bottom-left, a 90-degree clockwise rotated heart on the top-left, a 180-degree clockwise rotated heart on the top-right, and a 270-degree clockwise rotated heart on the bottom-right. The final Rune consists of four runes.] 7. [Function "quarter_turn_left" Makes a new Rune from a given Rune by turning it a quarter-turn in an anti-clockwise direction. Function prototype: quarter_turn_right(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example 1: "quarter_turn_left(heart)" rotates the heart shape rune 90 degrees in an anti-clockwise direction. Example 2: "quarter_turn_left(stack(r1, r2))" rotates the output of stack(r1, r2) 90 degrees in an anti-clockwise direction. ] 8. [Function "quarter_turn_right" makes a new Rune from a given Rune by turning it a quarter-turn around the center in a clockwise direction. Function prototype: quarter_turn_right(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example 1: "quarter_turn_right(heart)" rotates the heart shape rune 90 degrees in a clockwise direction. Example 2: "quarter_turn_right(stack(r1, r2))" rotates the output of stack(r1, r2) 90 degrees in a clockwise direction. ] 9. [Function "turn_upside_down" makes a new Rune from a given Rune by turning it upside-down. Function prototype: turn_upside_down(rune: Rune): Rune Prototype Description: It takes a Rune parameter as input and returns a Rune parameter. Example 1: "turn_upside_down(heart)" rotates a heart shape rune 180 degrees in a clockwise direction. Example 2: "turn_upside_down(stack(r1, r2))" rotates the output of stack(r1, r2) 180 degrees in a clockwise direction.] You must only use the Runes and functions declared above and avoid importing any module in your program. You can pass the output of each function as input to another function. For example, consider beside(stack(r2, r1), stack(r3, r4)). First, the inner stack functions get executed. r2 goes to the left of r1, and r3 goes to the left of r4. Then the output Rune of each stack works as input of beside function. meaning output of stak(r2, r1) goes on top of output of stack(r3,r4). Avoid hard coding. #Programming question# Write a function hook that takes a fraction "frac" as an input and creates a 'hook' pattern. The fraction input determines the size of the base of the hook. The output rune: [Imagine a rectangle divided into two horizontal sections. Each section is the height of a square. Top Section: This section is simply a filled square. Bottom Section: The bottom section is also the size of a square. However, it's divided into two equal parts vertically. The left side of this square is filled (so it looks like a rectangle that's half the width of the square). The right side of this square is blank or empty. So, if you place these two sections on top of one another, you get: A full square on top. Directly below it, on the left side, you have a half-filled square (a rectangle), and on the right side, it's empty. This forms a "hook" rune, with the hook part facing to the left. The overall rune is a square with two times the height of the original squares used to create it. Examples: hook(1): It's simply a square rune. hook(0): A filled square at the top. An empty or blank space at the bottom of the same size as the square. hook(1/2): A full square on top. Below that, on the right side, there's another filled square that's half the width of the full square. On the left side, it's empty. hook(1/5): A full square on top. Below that, on the right side, there's a very thin filled rectangle (only 1/5 the width of the square). The rest (4/5) to the right is empty.] You will only need to use the square and blank primitive runes and transform them to get the hook. Implement your function in the code below: "function hook(frac) { // your answer here } // Test show(hook(1/5));" #Sample Solution and feedback# 1. "function hook(frac) { return stack(square, quarter_turn_right( stack_frac(frac, square, blank))); } // Test show(hook(1/5));" - Great job! 2. "function hook(frac) { return frac === 1 ? square : frac === 0 ? stack(square,blank) : stack(square,beside_frac(1-frac, blank, square)); } show(hook(1/5));" - Excellent work! 3."function hook(frac) { return stack(square, quarter_turn_left( stack_frac(1-frac, blank, square))); } show(hook(1/5));" -Great job! 4."function hook(frac) { // your answer here return stack_frac(1/2,square, beside_frac(1-frac,blank,square)); } // Test show(hook(1/5));" -Good job, However stack_frac(1 / 2, etc) could have been simplified by merely using stack. """ - - prompt = raw_prompt <> "\n" <> answers_json - + # Get the llm_prompt from the first answer's question + llm_prompt = + answers + |> List.first() + |> Map.get(:question) + |> Map.get(:question) + |> Map.get("llm_prompt") + + # Combine prompts if llm_prompt exists + prompt = + if llm_prompt && llm_prompt != "" do + raw_prompt <> "Additional Instructions:\n\n" <> llm_prompt <> "\n\n" <> answers_json + else + raw_prompt <> "\n" <> answers_json + end input = %{ model: @model, @@ -209,11 +221,11 @@ defmodule CadetWeb.AICodeAnalysisController do ] - case HTTPoison.post(@openai_api_url, input, headers) do + case HTTPoison.post(@openai_api_url, input, headers, timeout: 60_000, recv_timeout: 60_000) do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> case Jason.decode(body) do {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> - log_comment(submission_id, question_id, raw_prompt, answers_json, response) + log_comment(submission_id, question_id, prompt, answers_json, response) comments_list = String.split(response, "|||") filtered_comments = Enum.filter(comments_list, fn comment -> @@ -223,16 +235,16 @@ defmodule CadetWeb.AICodeAnalysisController do json(conn, %{"comments" => filtered_comments}) {:error, _} -> - log_comment(submission_id, question_id, raw_prompt, answers_json, nil, "Failed to parse response from OpenAI API") + log_comment(submission_id, question_id, prompt, answers_json, nil, "Failed to parse response from OpenAI API") json(conn, %{"error" => "Failed to parse response from OpenAI API"}) end {:ok, %HTTPoison.Response{status_code: status, body: body}} -> - log_comment(submission_id, question_id, raw_prompt, answers_json, nil, "API request failed with status #{status}") + log_comment(submission_id, question_id, prompt, answers_json, nil, "API request failed with status #{status}") json(conn, %{"error" => "API request failed with status #{status}: #{body}"}) {:error, %HTTPoison.Error{reason: reason}} -> - log_comment(submission_id, question_id, raw_prompt, answers_json, nil, reason) + log_comment(submission_id, question_id, prompt, answers_json, nil, reason) json(conn, %{"error" => "HTTP request error: #{inspect(reason)}"}) end end diff --git a/test/support/ai_comments_test_helpers.ex b/test/support/ai_comments_test_helpers.ex new file mode 100644 index 000000000..a359075b5 --- /dev/null +++ b/test/support/ai_comments_test_helpers.ex @@ -0,0 +1,50 @@ +defmodule Cadet.AICommentsTestHelpers do + @moduledoc """ + Helper functions for testing AI comments functionality. + """ + + alias Cadet.Repo + alias Cadet.AIComments.AIComment + import Ecto.Query + + @doc """ + Gets the latest AI comment from the database. + """ + def get_latest_comment do + AIComment + |> first(order_by: [desc: :inserted_at]) + |> Repo.one() + end + + @doc """ + Gets all AI comments for a specific submission and question. + """ + def get_comments_for_submission(submission_id, question_id) do + from(c in AIComment, + where: c.submission_id == ^submission_id and c.question_id == ^question_id, + order_by: [desc: c.inserted_at] + ) + |> Repo.all() + end + + @doc """ + Reads the CSV log file and returns its contents. + """ + def read_csv_log do + log_file = "log/ai_comments.csv" + if File.exists?(log_file) do + File.read!(log_file) + else + "" + end + end + + @doc """ + Cleans up test artifacts. + """ + def cleanup_test_artifacts do + log_file = "log/ai_comments.csv" + File.rm(log_file) + Repo.delete_all(AIComment) + end +end From df34dbd79920224ed0865a97f867906a54412c5b Mon Sep 17 00:00:00 2001 From: Arul Date: Wed, 19 Mar 2025 14:40:20 +0800 Subject: [PATCH 09/77] feat: enabling/disabling of LLM feature by course level --- lib/cadet/courses/course.ex | 4 ++- .../controllers/generate_ai_comments.ex | 27 ++++++++++++++++--- .../20240320000000_add_llm_grading_access.exs | 15 +++++++++++ 3 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 priv/repo/migrations/20240320000000_add_llm_grading_access.exs diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index c74d23bd7..e3b3599ca 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -14,6 +14,7 @@ defmodule Cadet.Courses.Course do enable_achievements: boolean(), enable_sourcecast: boolean(), enable_stories: boolean(), + enable_llm_grading: boolean(), source_chapter: integer(), source_variant: String.t(), module_help_text: String.t(), @@ -28,6 +29,7 @@ defmodule Cadet.Courses.Course do field(:enable_achievements, :boolean, default: true) field(:enable_sourcecast, :boolean, default: true) field(:enable_stories, :boolean, default: false) + field(:enable_llm_grading, :boolean) field(:source_chapter, :integer) field(:source_variant, :string) field(:module_help_text, :string) @@ -42,7 +44,7 @@ defmodule Cadet.Courses.Course do @required_fields ~w(course_name viewable enable_game enable_achievements enable_sourcecast enable_stories source_chapter source_variant)a - @optional_fields ~w(course_short_name module_help_text)a + @optional_fields ~w(course_short_name module_help_text enable_llm_grading)a def changeset(course, params) do course diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index d96109da6..8d68e243a 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -6,10 +6,12 @@ defmodule CadetWeb.AICodeAnalysisController do alias Cadet.Assessments alias Cadet.AIComments + alias Cadet.Courses @openai_api_url "https://api.openai.com/v1/chat/completions" @model "gpt-4o" @api_key Application.get_env(:openai, :api_key) + @default_llm_grading false # For logging outputs to both database and file defp log_comment(submission_id, question_id, raw_prompt, answers_json, response, error \\ nil) do @@ -54,11 +56,26 @@ defmodule CadetWeb.AICodeAnalysisController do @doc """ Fetches the question details and answers based on submissionid and questionid and generates AI-generated comments. """ - def generate_ai_comments(conn, %{"submissionid" => submission_id, "questionid" => question_id}) + def generate_ai_comments(conn, %{"submissionid" => submission_id, "questionid" => question_id, "course_id" => course_id}) when is_ecto_id(submission_id) do - case Assessments.get_answers_in_submission(submission_id, question_id) do - {:ok, {answers, _assessment}} -> - analyze_code(conn, answers, submission_id, question_id) + # Check if LLM grading is enabled for this course (default to @default_llm_grading if nil) + case Courses.get_course_config(course_id) do + {:ok, course} -> + if course.enable_llm_grading == true || (course.enable_llm_grading == nil && @default_llm_grading == true) do + case Assessments.get_answers_in_submission(submission_id, question_id) do + {:ok, {answers, _assessment}} -> + analyze_code(conn, answers, submission_id, question_id) + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end + else + conn + |> put_status(:forbidden) + |> json(%{"error" => "LLM grading is not enabled for this course"}) + end {:error, {status, message}} -> conn @@ -274,6 +291,7 @@ defmodule CadetWeb.AICodeAnalysisController do produces("application/json") parameters do + courseId(:path, :integer, "course id", required: true) submissionId(:path, :integer, "submission id", required: true) questionId(:path, :integer, "question id", required: true) end @@ -282,6 +300,7 @@ defmodule CadetWeb.AICodeAnalysisController do response(400, "Invalid or missing parameter(s) or submission and/or question not found") response(401, "Unauthorized") response(403, "Forbidden") + response(403, "LLM grading is not enabled for this course") end swagger_path :save_final_comment do diff --git a/priv/repo/migrations/20240320000000_add_llm_grading_access.exs b/priv/repo/migrations/20240320000000_add_llm_grading_access.exs new file mode 100644 index 000000000..c5f337eee --- /dev/null +++ b/priv/repo/migrations/20240320000000_add_llm_grading_access.exs @@ -0,0 +1,15 @@ +defmodule Cadet.Repo.Migrations.AddLLMGradingAccess do + use Ecto.Migration + + def up do + alter table(:courses) do + add(:enable_llm_grading, :boolean, null: true) + end + end + + def down do + alter table(:courses) do + remove(:enable_llm_grading) + end + end +end From 0a25fa8e994fa795bbf65da61b62ee63f11aedb2 Mon Sep 17 00:00:00 2001 From: Arul Date: Wed, 19 Mar 2025 17:00:57 +0800 Subject: [PATCH 10/77] feat: added llm_grading boolean field to course creation API --- lib/cadet_web/controllers/courses_controller.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/cadet_web/controllers/courses_controller.ex b/lib/cadet_web/controllers/courses_controller.ex index e6555bd7a..5b88be0dc 100644 --- a/lib/cadet_web/controllers/courses_controller.ex +++ b/lib/cadet_web/controllers/courses_controller.ex @@ -56,6 +56,7 @@ defmodule CadetWeb.CoursesController do enable_achievements(:body, :boolean, "Enable achievements", required: true) enable_sourcecast(:body, :boolean, "Enable sourcecast", required: true) enable_stories(:body, :boolean, "Enable stories", required: true) + enable_llm_grading(:body, :boolean, "Enable LLM grading", required: false) source_chapter(:body, :number, "Default source chapter", required: true) source_variant(:body, Schema.ref(:SourceVariant), "Default source variant name", @@ -97,6 +98,7 @@ defmodule CadetWeb.CoursesController do enable_achievements(:boolean, "Enable achievements", required: true) enable_sourcecast(:boolean, "Enable sourcecast", required: true) enable_stories(:boolean, "Enable stories", required: true) + enable_llm_grading(:boolean, "Enable LLM grading", required: false) source_chapter(:integer, "Source Chapter number from 1 to 4", required: true) source_variant(Schema.ref(:SourceVariant), "Source Variant name", required: true) module_help_text(:string, "Module help text", required: true) @@ -111,6 +113,7 @@ defmodule CadetWeb.CoursesController do enable_achievements: true, enable_sourcecast: true, enable_stories: false, + enable_llm_grading: false, source_chapter: 1, source_variant: "default", module_help_text: "Help text", From 2723f5ae87ac66ad8ba6decd2728658ae52b9128 Mon Sep 17 00:00:00 2001 From: Arul Date: Wed, 26 Mar 2025 11:39:05 +0800 Subject: [PATCH 11/77] feat: added api key storage in courses & edit api key/enable llm grading --- lib/cadet/courses/course.ex | 4 ++- .../admin_courses_controller.ex | 2 ++ .../controllers/courses_controller.ex | 3 ++ .../controllers/generate_ai_comments.ex | 30 ++++++++++++------- lib/cadet_web/views/courses_view.ex | 2 ++ ...40320000001_add_llm_api_key_to_courses.exs | 15 ++++++++++ 6 files changed, 44 insertions(+), 12 deletions(-) create mode 100644 priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index e3b3599ca..eb1fcb386 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -15,6 +15,7 @@ defmodule Cadet.Courses.Course do enable_sourcecast: boolean(), enable_stories: boolean(), enable_llm_grading: boolean(), + llm_api_key: String.t() | nil, source_chapter: integer(), source_variant: String.t(), module_help_text: String.t(), @@ -30,6 +31,7 @@ defmodule Cadet.Courses.Course do field(:enable_sourcecast, :boolean, default: true) field(:enable_stories, :boolean, default: false) field(:enable_llm_grading, :boolean) + field(:llm_api_key, :string) field(:source_chapter, :integer) field(:source_variant, :string) field(:module_help_text, :string) @@ -44,7 +46,7 @@ defmodule Cadet.Courses.Course do @required_fields ~w(course_name viewable enable_game enable_achievements enable_sourcecast enable_stories source_chapter source_variant)a - @optional_fields ~w(course_short_name module_help_text enable_llm_grading)a + @optional_fields ~w(course_short_name module_help_text enable_llm_grading llm_api_key)a def changeset(course, params) do course diff --git a/lib/cadet_web/admin_controllers/admin_courses_controller.ex b/lib/cadet_web/admin_controllers/admin_courses_controller.ex index 7220a4d80..09d7d8616 100644 --- a/lib/cadet_web/admin_controllers/admin_courses_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_courses_controller.ex @@ -108,6 +108,8 @@ defmodule CadetWeb.AdminCoursesController do enable_achievements(:body, :boolean, "Enable achievements") enable_sourcecast(:body, :boolean, "Enable sourcecast") enable_stories(:body, :boolean, "Enable stories") + enable_llm_grading(:body, :boolean, "Enable LLM grading") + llm_api_key(:body, :string, "OpenAI API key for this course") sublanguage(:body, Schema.ref(:AdminSublanguage), "sublanguage object") module_help_text(:body, :string, "Module help text") end diff --git a/lib/cadet_web/controllers/courses_controller.ex b/lib/cadet_web/controllers/courses_controller.ex index 5b88be0dc..c717e3f06 100644 --- a/lib/cadet_web/controllers/courses_controller.ex +++ b/lib/cadet_web/controllers/courses_controller.ex @@ -57,6 +57,7 @@ defmodule CadetWeb.CoursesController do enable_sourcecast(:body, :boolean, "Enable sourcecast", required: true) enable_stories(:body, :boolean, "Enable stories", required: true) enable_llm_grading(:body, :boolean, "Enable LLM grading", required: false) + llm_api_key(:body, :string, "OpenAI API key for this course", required: false) source_chapter(:body, :number, "Default source chapter", required: true) source_variant(:body, Schema.ref(:SourceVariant), "Default source variant name", @@ -99,6 +100,7 @@ defmodule CadetWeb.CoursesController do enable_sourcecast(:boolean, "Enable sourcecast", required: true) enable_stories(:boolean, "Enable stories", required: true) enable_llm_grading(:boolean, "Enable LLM grading", required: false) + llm_api_key(:string, "OpenAI API key for this course", required: false) source_chapter(:integer, "Source Chapter number from 1 to 4", required: true) source_variant(Schema.ref(:SourceVariant), "Source Variant name", required: true) module_help_text(:string, "Module help text", required: true) @@ -114,6 +116,7 @@ defmodule CadetWeb.CoursesController do enable_sourcecast: true, enable_stories: false, enable_llm_grading: false, + llm_api_key: "sk-1234567890", source_chapter: 1, source_variant: "default", module_help_text: "Help text", diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 8d68e243a..d8d53fd4a 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -10,7 +10,6 @@ defmodule CadetWeb.AICodeAnalysisController do @openai_api_url "https://api.openai.com/v1/chat/completions" @model "gpt-4o" - @api_key Application.get_env(:openai, :api_key) @default_llm_grading false # For logging outputs to both database and file @@ -62,14 +61,23 @@ defmodule CadetWeb.AICodeAnalysisController do case Courses.get_course_config(course_id) do {:ok, course} -> if course.enable_llm_grading == true || (course.enable_llm_grading == nil && @default_llm_grading == true) do - case Assessments.get_answers_in_submission(submission_id, question_id) do - {:ok, {answers, _assessment}} -> - analyze_code(conn, answers, submission_id, question_id) - - {:error, {status, message}} -> - conn - |> put_status(status) - |> text(message) + # Get API key from course config or fall back to environment variable + api_key = course.llm_api_key || Application.get_env(:openai, :api_key) + + if is_nil(api_key) do + conn + |> put_status(:internal_server_error) + |> json(%{"error" => "No OpenAI API key configured"}) + else + case Assessments.get_answers_in_submission(submission_id, question_id) do + {:ok, {answers, _assessment}} -> + analyze_code(conn, answers, submission_id, question_id, api_key) + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end end else conn @@ -129,7 +137,7 @@ defmodule CadetWeb.AICodeAnalysisController do """ end - defp analyze_code(conn, answers, submission_id, question_id) do + defp analyze_code(conn, answers, submission_id, question_id, api_key) do answers_json = answers |> Enum.map(fn answer -> @@ -233,7 +241,7 @@ defmodule CadetWeb.AICodeAnalysisController do } |> Jason.encode!() headers = [ - {"Authorization", "Bearer #{@api_key}"}, + {"Authorization", "Bearer #{api_key}"}, {"Content-Type", "application/json"} ] diff --git a/lib/cadet_web/views/courses_view.ex b/lib/cadet_web/views/courses_view.ex index a6ae9c4fa..819f4aebd 100644 --- a/lib/cadet_web/views/courses_view.ex +++ b/lib/cadet_web/views/courses_view.ex @@ -12,6 +12,8 @@ defmodule CadetWeb.CoursesView do enableAchievements: :enable_achievements, enableSourcecast: :enable_sourcecast, enableStories: :enable_stories, + enableLlmGrading: :enable_llm_grading, + llmApiKey: :llm_api_key, sourceChapter: :source_chapter, sourceVariant: :source_variant, moduleHelpText: :module_help_text, diff --git a/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs new file mode 100644 index 000000000..e752889b7 --- /dev/null +++ b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs @@ -0,0 +1,15 @@ +defmodule Cadet.Repo.Migrations.AddLlmApiKeyToCourses do + use Ecto.Migration + + def up do + alter table(:courses) do + add(:llm_api_key, :string, null: true) + end + end + + def down do + alter table(:courses) do + remove(:llm_api_key) + end + end +end From 02f7ed178a2df244713462d89546ed36614fd245 Mon Sep 17 00:00:00 2001 From: Arul Date: Wed, 2 Apr 2025 16:32:10 +0800 Subject: [PATCH 12/77] feat: encryption for llm_api_key --- lib/cadet/courses/course.ex | 37 ++++++++++++++++ .../controllers/generate_ai_comments.ex | 42 +++++++++++++++++-- ...0402030934_increase_llm_api_key_length.exs | 9 ++++ 3 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index eb1fcb386..2261ba10d 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -48,11 +48,48 @@ defmodule Cadet.Courses.Course do enable_achievements enable_sourcecast enable_stories source_chapter source_variant)a @optional_fields ~w(course_short_name module_help_text enable_llm_grading llm_api_key)a + @spec changeset( + {map(), map()} + | %{ + :__struct__ => atom() | %{:__changeset__ => map(), optional(any()) => any()}, + optional(atom()) => any() + }, + %{optional(:__struct__) => none(), optional(atom() | binary()) => any()} + ) :: Ecto.Changeset.t() def changeset(course, params) do course |> cast(params, @required_fields ++ @optional_fields) |> validate_required(@required_fields) |> validate_sublanguage_combination(params) + |> put_encrypted_llm_api_key() + end + + def put_encrypted_llm_api_key(changeset) do + if llm_api_key = get_change(changeset, :llm_api_key) do + if is_binary(llm_api_key) and llm_api_key != "" do + secret = Application.get_env(:openai, :encryption_key) + if is_binary(secret) and byte_size(secret) >= 16 do + # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256 + key = binary_part(secret, 0, min(32, byte_size(secret))) + # Use AES in GCM mode for encryption + iv = :crypto.strong_rand_bytes(16) + {ciphertext, tag} = :crypto.crypto_one_time_aead( + :aes_gcm, key, iv, llm_api_key, "", true + ) + # Store both the IV, ciphertext and tag + encrypted = iv <> tag <> ciphertext + put_change(changeset, :llm_api_key, Base.encode64(encrypted)) + else + add_error(changeset, :llm_api_key, "encryption key not configured properly") + end + else + # If empty string or nil is provided, don't encrypt but don't add error + changeset + end + else + # The key is not being changed, so we need to preserve the existing value + put_change(changeset, :llm_api_key, changeset.data.llm_api_key) + end end # Validates combination of Source chapter and variant diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index d8d53fd4a..6f0f2bf64 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -10,7 +10,7 @@ defmodule CadetWeb.AICodeAnalysisController do @openai_api_url "https://api.openai.com/v1/chat/completions" @model "gpt-4o" - @default_llm_grading false + @default_llm_grading false # To set whether LLM grading is enabled across Source Academy # For logging outputs to both database and file defp log_comment(submission_id, question_id, raw_prompt, answers_json, response, error \\ nil) do @@ -60,9 +60,11 @@ defmodule CadetWeb.AICodeAnalysisController do # Check if LLM grading is enabled for this course (default to @default_llm_grading if nil) case Courses.get_course_config(course_id) do {:ok, course} -> - if course.enable_llm_grading == true || (course.enable_llm_grading == nil && @default_llm_grading == true) do + if course.enable_llm_grading || @default_llm_grading do + Logger.info("LLM Api key: #{course.llm_api_key}") # Get API key from course config or fall back to environment variable - api_key = course.llm_api_key || Application.get_env(:openai, :api_key) + decrypted_api_key = decrypt_llm_api_key(course.llm_api_key) + api_key = decrypted_api_key || Application.get_env(:openai, :api_key) if is_nil(api_key) do conn @@ -349,4 +351,38 @@ defmodule CadetWeb.AICodeAnalysisController do end } end + + defp decrypt_llm_api_key(nil), do: nil + defp decrypt_llm_api_key(encrypted_key) do + try do + # Get the encryption key + secret = Application.get_env(:openai, :encryption_key) + + if is_binary(secret) and byte_size(secret) >= 16 do + # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256 + key = binary_part(secret, 0, min(32, byte_size(secret))) + + # Decode the base64 string + decoded = Base.decode64!(encrypted_key) + + # Extract IV, tag and ciphertext + iv = binary_part(decoded, 0, 16) + tag = binary_part(decoded, 16, 16) + ciphertext = binary_part(decoded, 32, byte_size(decoded) - 32) + + # Decrypt + case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do + plain_text when is_binary(plain_text) -> plain_text + _ -> nil + end + else + Logger.error("Encryption key not configured properly") + nil + end + rescue + e -> + Logger.error("Error decrypting LLM API key: #{inspect(e)}") + nil + end + end end diff --git a/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs b/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs new file mode 100644 index 000000000..295c83eea --- /dev/null +++ b/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs @@ -0,0 +1,9 @@ +defmodule Cadet.Repo.Migrations.IncreaseLlmApiKeyLength do + use Ecto.Migration + + def change do + alter table(:courses) do + modify :llm_api_key, :text + end + end +end From cb3498444727178d8776312cedb44840b9f49753 Mon Sep 17 00:00:00 2001 From: Arul Date: Sun, 6 Apr 2025 01:41:02 +0800 Subject: [PATCH 13/77] feat: added final comment editing route --- lib/cadet/ai_comments.ex | 29 +++++++++++-- .../controllers/generate_ai_comments.ex | 42 +++++++++---------- lib/cadet_web/router.ex | 6 +++ 3 files changed, 51 insertions(+), 26 deletions(-) diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index 3f42c9004..c0ddcde91 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -18,14 +18,27 @@ defmodule Cadet.AIComments do def get_ai_comment!(id), do: Repo.get!(AIComment, id) @doc """ - Gets AI comments for a specific submission and question. + Retrieves an AI comment for a specific submission and question. + Returns `nil` if no comment exists. """ def get_ai_comments_for_submission(submission_id, question_id) do + Repo.one( + from c in AIComment, + where: c.submission_id == ^submission_id and c.question_id == ^question_id + ) + end + + @doc """ + Retrieves the latest AI comment for a specific submission and question. + Returns `nil` if no comment exists. + """ + def get_latest_ai_comment(submission_id, question_id) do from(c in AIComment, where: c.submission_id == ^submission_id and c.question_id == ^question_id, - order_by: [desc: c.inserted_at] + order_by: [desc: c.inserted_at], + limit: 1 ) - |> Repo.all() + |> Repo.one() end @doc """ @@ -47,4 +60,14 @@ defmodule Cadet.AIComments do |> Repo.update() end end + + @doc """ + Updates an existing AI comment with new attributes. + """ + def update_ai_comment(id, attrs) do + id + |> get_ai_comment!() + |> AIComment.changeset(attrs) + |> Repo.update() + end end diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 6f0f2bf64..652645b0a 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -25,30 +25,26 @@ defmodule CadetWeb.AICodeAnalysisController do inserted_at: NaiveDateTime.utc_now() } - case AIComments.create_ai_comment(attrs) do - {:ok, comment} -> {:ok, comment} - {:error, changeset} -> - Logger.error("Failed to log AI comment to database: #{inspect(changeset.errors)}") - {:error, changeset} - end - - # Log to file - try do - log_file = "log/ai_comments.csv" - File.mkdir_p!("log") - - timestamp = NaiveDateTime.utc_now() |> NaiveDateTime.to_string() - raw_prompt_str = Jason.encode!(raw_prompt) |> String.replace("\"", "\"\"") - answers_json_str = answers_json |> String.replace("\"", "\"\"") - response_str = if is_nil(response), do: "", else: response |> String.replace("\"", "\"\"") - error_str = if is_nil(error), do: "", else: error |> String.replace("\"", "\"\"") - - csv_row = "\"#{timestamp}\",\"#{submission_id}\",\"#{question_id}\",\"#{raw_prompt_str}\",\"#{answers_json_str}\",\"#{response_str}\",\"#{error_str}\"\n" + # Check if a comment already exists for the given submission_id and question_id + case AIComments.get_latest_ai_comment(submission_id, question_id) do + nil -> + # If no existing comment, create a new one + case AIComments.create_ai_comment(attrs) do + {:ok, comment} -> {:ok, comment} + {:error, changeset} -> + Logger.error("Failed to log AI comment to database: #{inspect(changeset.errors)}") + {:error, changeset} + end - File.write!(log_file, csv_row, [:append]) - rescue - e -> - Logger.error("Failed to log AI comment to file: #{inspect(e)}") + existing_comment -> + # If a comment exists, update it with the new data + updated_attrs = Map.merge(existing_comment, attrs) + case AIComments.update_ai_comment(existing_comment.id, updated_attrs) do + {:ok, updated_comment} -> {:ok, updated_comment} + {:error, changeset} -> + Logger.error("Failed to update AI comment in database: #{inspect(changeset.errors)}") + {:error, changeset} + end end end diff --git a/lib/cadet_web/router.ex b/lib/cadet_web/router.ex index 3e8c1a155..18aead900 100644 --- a/lib/cadet_web/router.ex +++ b/lib/cadet_web/router.ex @@ -213,6 +213,12 @@ defmodule CadetWeb.Router do :autograde_answer ) + post( + "/save-final-comment/:submissionid/:questionid", + AICodeAnalysisController, + :save_final_comment + ) + get("/users", AdminUserController, :index) get("/users/teamformation", AdminUserController, :get_students) put("/users", AdminUserController, :upsert_users_and_groups) From 09a7b093618ea0bfc09282cf38789dab8204c3ae Mon Sep 17 00:00:00 2001 From: Arul Date: Sun, 6 Apr 2025 13:56:43 +0800 Subject: [PATCH 14/77] feat: added logging of chosen comments --- lib/cadet/ai_comments.ex | 20 +++++++++++++++++++ lib/cadet/ai_comments/ai_comment.ex | 2 +- .../controllers/generate_ai_comments.ex | 16 +++++++++++++++ lib/cadet_web/router.ex | 6 ++++++ ...6053008_update_comment_chosen_to_array.exs | 17 ++++++++++++++++ 5 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index c0ddcde91..cbdae755c 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -70,4 +70,24 @@ defmodule Cadet.AIComments do |> AIComment.changeset(attrs) |> Repo.update() end + + @doc """ + Updates the chosen comments for a specific submission and question. + Accepts an array of comments and replaces the existing array in the database. + """ + def update_chosen_comments(submission_id, question_id, new_comments) do + from(c in AIComment, + where: c.submission_id == ^submission_id and c.question_id == ^question_id, + order_by: [desc: c.inserted_at], + limit: 1 + ) + |> Repo.one() + |> case do + nil -> {:error, :not_found} + comment -> + comment + |> AIComment.changeset(%{comment_chosen: new_comments}) + |> Repo.update() + end + end end diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex index 575d1cc10..55c01802c 100644 --- a/lib/cadet/ai_comments/ai_comment.ex +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -9,7 +9,7 @@ defmodule Cadet.AIComments.AIComment do field :answers_json, :string field :response, :string field :error, :string - field :comment_chosen, :string + field :comment_chosen, {:array, :string} field :final_comment, :string timestamps() diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 652645b0a..45b5e830b 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -286,6 +286,22 @@ defmodule CadetWeb.AICodeAnalysisController do end end + @doc """ + Saves the chosen comments for a submission and question. + Accepts an array of comments in the request body. + """ + def save_chosen_comments(conn, %{"submissionid" => submission_id, "questionid" => question_id, "comments" => comments}) do + case AIComments.update_chosen_comments(submission_id, question_id, comments) do + {:ok, _updated_comment} -> + json(conn, %{"status" => "success"}) + + {:error, changeset} -> + conn + |> put_status(:unprocessable_entity) + |> json(%{"error" => "Failed to save chosen comments"}) + end + end + swagger_path :generate_ai_comments do post("/courses/{courseId}/admin/generate-comments/{submissionId}/{questionId}") diff --git a/lib/cadet_web/router.ex b/lib/cadet_web/router.ex index 18aead900..c64631cf8 100644 --- a/lib/cadet_web/router.ex +++ b/lib/cadet_web/router.ex @@ -219,6 +219,12 @@ defmodule CadetWeb.Router do :save_final_comment ) + post( + "/save-chosen-comments/:submissionid/:questionid", + AICodeAnalysisController, + :save_chosen_comments + ) + get("/users", AdminUserController, :index) get("/users/teamformation", AdminUserController, :get_students) put("/users", AdminUserController, :upsert_users_and_groups) diff --git a/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs b/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs new file mode 100644 index 000000000..112f8b4c2 --- /dev/null +++ b/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs @@ -0,0 +1,17 @@ +defmodule Cadet.Repo.Migrations.UpdateCommentChosenToArray do + use Ecto.Migration + + def change do + alter table(:ai_comment_logs) do + add :comment_chosen_temp, {:array, :string}, default: [] + end + + execute "UPDATE ai_comment_logs SET comment_chosen_temp = ARRAY[comment_chosen]" + + alter table(:ai_comment_logs) do + remove :comment_chosen + end + + rename table(:ai_comment_logs), :comment_chosen_temp, to: :comment_chosen + end +end From ed44a7e7ca88d6634fae5b3412f164145a927800 Mon Sep 17 00:00:00 2001 From: Arul Date: Sun, 6 Apr 2025 14:30:24 +0800 Subject: [PATCH 15/77] fix: bugs when certain fields were missing --- .../controllers/generate_ai_comments.ex | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 45b5e830b..fcbf324ca 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -37,8 +37,8 @@ defmodule CadetWeb.AICodeAnalysisController do end existing_comment -> - # If a comment exists, update it with the new data - updated_attrs = Map.merge(existing_comment, attrs) + # Convert the existing comment struct to a map before merging + updated_attrs = Map.merge(Map.from_struct(existing_comment), attrs) case AIComments.update_ai_comment(existing_comment.id, updated_attrs) do {:ok, updated_comment} -> {:ok, updated_comment} {:error, changeset} -> @@ -114,27 +114,37 @@ defmodule CadetWeb.AICodeAnalysisController do defp format_answer(answer) do """ - **Question ID: #{answer["question"]["id"]}** + **Question ID: #{answer["question"]["id"] || "N/A"}** **Question:** - #{answer["question"]["content"]} + #{answer["question"]["content"] || "N/A"} **Solution:** ``` - #{answer["question"]["solution"]} + #{answer["question"]["solution"] || "N/A"} ``` **Answer:** ``` - #{answer["answer"]["code"]} + #{answer["answer"]["code"] || "N/A"} ``` - **Autograding Status:** #{answer["autograding_status"]} - **Autograding Results:** #{answer["autograding_results"]} + **Autograding Status:** #{answer["autograding_status"] || "N/A"} + **Autograding Results:** #{format_autograding_results(answer["autograding_results"])} **Comments:** #{answer["comments"] || "None"} """ end + defp format_autograding_results(nil), do: "N/A" + defp format_autograding_results(results) when is_list(results) do + results + |> Enum.map(fn result -> + "Error: #{result["errorMessage"] || "N/A"}, Type: #{result["errorType"] || "N/A"}" + end) + |> Enum.join("; ") + end + defp format_autograding_results(results), do: inspect(results) + defp analyze_code(conn, answers, submission_id, question_id, api_key) do answers_json = answers From 3715368fad436211aee3404784de325b4753fd96 Mon Sep 17 00:00:00 2001 From: Arul Date: Sun, 6 Apr 2025 14:36:59 +0800 Subject: [PATCH 16/77] feat: updated tests --- .../ai_code_analysis_controller_test.exs | 49 ++++++++++++++++++ test/support/ai_comments_test_helpers.ex | 50 ------------------- 2 files changed, 49 insertions(+), 50 deletions(-) delete mode 100644 test/support/ai_comments_test_helpers.ex diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs index 559c4ea9e..54492c1ee 100644 --- a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -121,4 +121,53 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do assert response["error"] == "Failed to save final comment" end end + + describe "save_chosen_comments" do + test "successfully saves chosen comments", %{conn: conn} do + # First create a comment entry + submission_id = 123 + question_id = 456 + raw_prompt = "Test prompt" + answers_json = ~s({"test": "data"}) + response = "Comment 1|||Comment 2|||Comment 3" + + {:ok, _comment} = AIComments.create_ai_comment(%{ + submission_id: submission_id, + question_id: question_id, + raw_prompt: raw_prompt, + answers_json: answers_json, + response: response + }) + + # Now save the chosen comments + chosen_comments = ["Comment 1", "Comment 2"] + response = + conn + |> post(Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), %{ + comments: chosen_comments + }) + |> json_response(200) + + assert response["status"] == "success" + + # Verify the chosen comments were saved + comment = Repo.get_by(AIComment, submission_id: submission_id, question_id: question_id) + assert comment.comment_chosen == chosen_comments + end + + test "returns error when no comment exists", %{conn: conn} do + submission_id = 999 + question_id = 888 + chosen_comments = ["Comment 1", "Comment 2"] + + response = + conn + |> post(Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), %{ + comments: chosen_comments + }) + |> json_response(422) + + assert response["error"] == "Failed to save chosen comments" + end + end end diff --git a/test/support/ai_comments_test_helpers.ex b/test/support/ai_comments_test_helpers.ex deleted file mode 100644 index a359075b5..000000000 --- a/test/support/ai_comments_test_helpers.ex +++ /dev/null @@ -1,50 +0,0 @@ -defmodule Cadet.AICommentsTestHelpers do - @moduledoc """ - Helper functions for testing AI comments functionality. - """ - - alias Cadet.Repo - alias Cadet.AIComments.AIComment - import Ecto.Query - - @doc """ - Gets the latest AI comment from the database. - """ - def get_latest_comment do - AIComment - |> first(order_by: [desc: :inserted_at]) - |> Repo.one() - end - - @doc """ - Gets all AI comments for a specific submission and question. - """ - def get_comments_for_submission(submission_id, question_id) do - from(c in AIComment, - where: c.submission_id == ^submission_id and c.question_id == ^question_id, - order_by: [desc: c.inserted_at] - ) - |> Repo.all() - end - - @doc """ - Reads the CSV log file and returns its contents. - """ - def read_csv_log do - log_file = "log/ai_comments.csv" - if File.exists?(log_file) do - File.read!(log_file) - else - "" - end - end - - @doc """ - Cleans up test artifacts. - """ - def cleanup_test_artifacts do - log_file = "log/ai_comments.csv" - File.rm(log_file) - Repo.delete_all(AIComment) - end -end From 5bfe276df8508e6487bebd873288a6ab819a4678 Mon Sep 17 00:00:00 2001 From: Arul Date: Sun, 6 Apr 2025 15:54:43 +0800 Subject: [PATCH 17/77] formatting --- lib/cadet/ai_comments.ex | 11 +- lib/cadet/ai_comments/ai_comment.ex | 27 +- lib/cadet/courses/course.ex | 15 +- .../controllers/generate_ai_comments.ex | 270 ++++++++++-------- ...0402030934_increase_llm_api_key_length.exs | 2 +- ...6053008_update_comment_chosen_to_array.exs | 8 +- .../ai_code_analysis_controller_test.exs | 76 +++-- 7 files changed, 247 insertions(+), 162 deletions(-) diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index cbdae755c..eaf91a344 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -23,8 +23,9 @@ defmodule Cadet.AIComments do """ def get_ai_comments_for_submission(submission_id, question_id) do Repo.one( - from c in AIComment, + from(c in AIComment, where: c.submission_id == ^submission_id and c.question_id == ^question_id + ) ) end @@ -53,7 +54,9 @@ defmodule Cadet.AIComments do ) |> Repo.one() |> case do - nil -> {:error, :not_found} + nil -> + {:error, :not_found} + comment -> comment |> AIComment.changeset(%{final_comment: final_comment}) @@ -83,7 +86,9 @@ defmodule Cadet.AIComments do ) |> Repo.one() |> case do - nil -> {:error, :not_found} + nil -> + {:error, :not_found} + comment -> comment |> AIComment.changeset(%{comment_chosen: new_comments}) diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex index 55c01802c..0cefe9530 100644 --- a/lib/cadet/ai_comments/ai_comment.ex +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -3,21 +3,30 @@ defmodule Cadet.AIComments.AIComment do import Ecto.Changeset schema "ai_comment_logs" do - field :submission_id, :integer - field :question_id, :integer - field :raw_prompt, :string - field :answers_json, :string - field :response, :string - field :error, :string - field :comment_chosen, {:array, :string} - field :final_comment, :string + field(:submission_id, :integer) + field(:question_id, :integer) + field(:raw_prompt, :string) + field(:answers_json, :string) + field(:response, :string) + field(:error, :string) + field(:comment_chosen, {:array, :string}) + field(:final_comment, :string) timestamps() end def changeset(ai_comment, attrs) do ai_comment - |> cast(attrs, [:submission_id, :question_id, :raw_prompt, :answers_json, :response, :error, :comment_chosen, :final_comment]) + |> cast(attrs, [ + :submission_id, + :question_id, + :raw_prompt, + :answers_json, + :response, + :error, + :comment_chosen, + :final_comment + ]) |> validate_required([:submission_id, :question_id, :raw_prompt, :answers_json]) end end diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index 2261ba10d..cf6170cfb 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -68,14 +68,23 @@ defmodule Cadet.Courses.Course do if llm_api_key = get_change(changeset, :llm_api_key) do if is_binary(llm_api_key) and llm_api_key != "" do secret = Application.get_env(:openai, :encryption_key) + if is_binary(secret) and byte_size(secret) >= 16 do # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256 key = binary_part(secret, 0, min(32, byte_size(secret))) # Use AES in GCM mode for encryption iv = :crypto.strong_rand_bytes(16) - {ciphertext, tag} = :crypto.crypto_one_time_aead( - :aes_gcm, key, iv, llm_api_key, "", true - ) + + {ciphertext, tag} = + :crypto.crypto_one_time_aead( + :aes_gcm, + key, + iv, + llm_api_key, + "", + true + ) + # Store both the IV, ciphertext and tag encrypted = iv <> tag <> ciphertext put_change(changeset, :llm_api_key, Base.encode64(encrypted)) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index fcbf324ca..48b81bc6b 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -10,7 +10,8 @@ defmodule CadetWeb.AICodeAnalysisController do @openai_api_url "https://api.openai.com/v1/chat/completions" @model "gpt-4o" - @default_llm_grading false # To set whether LLM grading is enabled across Source Academy + # To set whether LLM grading is enabled across Source Academy + @default_llm_grading false # For logging outputs to both database and file defp log_comment(submission_id, question_id, raw_prompt, answers_json, response, error \\ nil) do @@ -30,7 +31,9 @@ defmodule CadetWeb.AICodeAnalysisController do nil -> # If no existing comment, create a new one case AIComments.create_ai_comment(attrs) do - {:ok, comment} -> {:ok, comment} + {:ok, comment} -> + {:ok, comment} + {:error, changeset} -> Logger.error("Failed to log AI comment to database: #{inspect(changeset.errors)}") {:error, changeset} @@ -39,8 +42,11 @@ defmodule CadetWeb.AICodeAnalysisController do existing_comment -> # Convert the existing comment struct to a map before merging updated_attrs = Map.merge(Map.from_struct(existing_comment), attrs) + case AIComments.update_ai_comment(existing_comment.id, updated_attrs) do - {:ok, updated_comment} -> {:ok, updated_comment} + {:ok, updated_comment} -> + {:ok, updated_comment} + {:error, changeset} -> Logger.error("Failed to update AI comment in database: #{inspect(changeset.errors)}") {:error, changeset} @@ -51,43 +57,47 @@ defmodule CadetWeb.AICodeAnalysisController do @doc """ Fetches the question details and answers based on submissionid and questionid and generates AI-generated comments. """ - def generate_ai_comments(conn, %{"submissionid" => submission_id, "questionid" => question_id, "course_id" => course_id}) - when is_ecto_id(submission_id) do - # Check if LLM grading is enabled for this course (default to @default_llm_grading if nil) - case Courses.get_course_config(course_id) do - {:ok, course} -> - if course.enable_llm_grading || @default_llm_grading do - Logger.info("LLM Api key: #{course.llm_api_key}") - # Get API key from course config or fall back to environment variable - decrypted_api_key = decrypt_llm_api_key(course.llm_api_key) - api_key = decrypted_api_key || Application.get_env(:openai, :api_key) - - if is_nil(api_key) do - conn - |> put_status(:internal_server_error) - |> json(%{"error" => "No OpenAI API key configured"}) - else - case Assessments.get_answers_in_submission(submission_id, question_id) do - {:ok, {answers, _assessment}} -> - analyze_code(conn, answers, submission_id, question_id, api_key) - - {:error, {status, message}} -> - conn - |> put_status(status) - |> text(message) - end - end - else + def generate_ai_comments(conn, %{ + "submissionid" => submission_id, + "questionid" => question_id, + "course_id" => course_id + }) + when is_ecto_id(submission_id) do + # Check if LLM grading is enabled for this course (default to @default_llm_grading if nil) + case Courses.get_course_config(course_id) do + {:ok, course} -> + if course.enable_llm_grading || @default_llm_grading do + Logger.info("LLM Api key: #{course.llm_api_key}") + # Get API key from course config or fall back to environment variable + decrypted_api_key = decrypt_llm_api_key(course.llm_api_key) + api_key = decrypted_api_key || Application.get_env(:openai, :api_key) + + if is_nil(api_key) do conn - |> put_status(:forbidden) - |> json(%{"error" => "LLM grading is not enabled for this course"}) + |> put_status(:internal_server_error) + |> json(%{"error" => "No OpenAI API key configured"}) + else + case Assessments.get_answers_in_submission(submission_id, question_id) do + {:ok, {answers, _assessment}} -> + analyze_code(conn, answers, submission_id, question_id, api_key) + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end end - - {:error, {status, message}} -> + else conn - |> put_status(status) - |> text(message) - end + |> put_status(:forbidden) + |> json(%{"error" => "LLM grading is not enabled for this course"}) + end + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end end defp transform_answers(answers) do @@ -136,6 +146,7 @@ defmodule CadetWeb.AICodeAnalysisController do end defp format_autograding_results(nil), do: "N/A" + defp format_autograding_results(results) when is_list(results) do results |> Enum.map(fn result -> @@ -143,6 +154,7 @@ defmodule CadetWeb.AICodeAnalysisController do end) |> Enum.join("; ") end + defp format_autograding_results(results), do: inspect(results) defp analyze_code(conn, answers, submission_id, question_id, api_key) do @@ -164,6 +176,7 @@ defmodule CadetWeb.AICodeAnalysisController do llm_prompt: nil } end + answer |> Map.from_struct() |> Map.take([ @@ -171,89 +184,91 @@ defmodule CadetWeb.AICodeAnalysisController do :comments, :autograding_status, :autograding_results, - :answer, + :answer ]) |> Map.put(:question, question_data) end) |> Jason.encode!() |> format_answers() - raw_prompt = """ - The code below is written in Source, a variant of JavaScript that comes with a rich set of built-in constants and functions. Below is a summary of some key built-in entities available in Source: - - Constants: - - Infinity: The special number value representing infinity. - - NaN: The special number value for "not a number." - - undefined: The special value for an undefined variable. - - math_PI: The constant π (approximately 3.14159). - - math_E: Euler's number (approximately 2.71828). - - Functions: - - __access_export__(exports, lookup_name): Searches for a name in an exports data structure. - - accumulate(f, initial, xs): Reduces a list by applying a binary function from right-to-left. - - append(xs, ys): Appends list ys to the end of list xs. - - char_at(s, i): Returns the character at index i of string s. - - display(v, s): Displays value v (optionally preceded by string s) in the console. - - filter(pred, xs): Returns a new list with elements of xs that satisfy the predicate pred. - - for_each(f, xs): Applies function f to each element of the list xs. - - get_time(): Returns the current time in milliseconds. - - is_list(xs): Checks whether xs is a proper list. - - length(xs): Returns the number of elements in list xs. - - list(...): Constructs a list from the provided values. - - map(f, xs): Applies function f to each element of list xs. - - math_abs(x): Returns the absolute value of x. - - math_ceil(x): Rounds x up to the nearest integer. - - math_floor(x): Rounds x down to the nearest integer. - - pair(x, y): A primitive function that makes a pair whose head (first component) is x and whose tail (second component) is y. - - head(xs): Returns the first element of pair xs. - - tail(xs): Returns the second element of pair xs. - - math_random(): Returns a random number between 0 (inclusive) and 1 (exclusive). - - (For a full list of built-in functions and constants, refer to the Source documentation.) - - Analyze the following submitted answers and provide detailed feedback on correctness, readability, efficiency, and possible improvements. Your evaluation should consider both standard JavaScript features and the additional built-in functions unique to Source. - - Provide between 3 and 5 concise comment suggestions, each under 200 words. - - Your output must include only the comment suggestions, separated exclusively by triple pipes ("|||") with no spaces before or after the pipes, and without any additional formatting, bullet points, or extra text. - - Comments and documentation in the code are not necessary for the code, do not penalise based on that, do not suggest to add comments as well. - - Follow the XP scoring guideline provided below in the question prompt, do not be too harsh! - - For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." - """ - # Get the llm_prompt from the first answer's question - llm_prompt = - answers - |> List.first() - |> Map.get(:question) - |> Map.get(:question) - |> Map.get("llm_prompt") - - # Combine prompts if llm_prompt exists - prompt = - if llm_prompt && llm_prompt != "" do - raw_prompt <> "Additional Instructions:\n\n" <> llm_prompt <> "\n\n" <> answers_json - else - raw_prompt <> "\n" <> answers_json - end + raw_prompt = """ + The code below is written in Source, a variant of JavaScript that comes with a rich set of built-in constants and functions. Below is a summary of some key built-in entities available in Source: + + Constants: + - Infinity: The special number value representing infinity. + - NaN: The special number value for "not a number." + - undefined: The special value for an undefined variable. + - math_PI: The constant π (approximately 3.14159). + - math_E: Euler's number (approximately 2.71828). + + Functions: + - __access_export__(exports, lookup_name): Searches for a name in an exports data structure. + - accumulate(f, initial, xs): Reduces a list by applying a binary function from right-to-left. + - append(xs, ys): Appends list ys to the end of list xs. + - char_at(s, i): Returns the character at index i of string s. + - display(v, s): Displays value v (optionally preceded by string s) in the console. + - filter(pred, xs): Returns a new list with elements of xs that satisfy the predicate pred. + - for_each(f, xs): Applies function f to each element of the list xs. + - get_time(): Returns the current time in milliseconds. + - is_list(xs): Checks whether xs is a proper list. + - length(xs): Returns the number of elements in list xs. + - list(...): Constructs a list from the provided values. + - map(f, xs): Applies function f to each element of list xs. + - math_abs(x): Returns the absolute value of x. + - math_ceil(x): Rounds x up to the nearest integer. + - math_floor(x): Rounds x down to the nearest integer. + - pair(x, y): A primitive function that makes a pair whose head (first component) is x and whose tail (second component) is y. + - head(xs): Returns the first element of pair xs. + - tail(xs): Returns the second element of pair xs. + - math_random(): Returns a random number between 0 (inclusive) and 1 (exclusive). + + (For a full list of built-in functions and constants, refer to the Source documentation.) + + Analyze the following submitted answers and provide detailed feedback on correctness, readability, efficiency, and possible improvements. Your evaluation should consider both standard JavaScript features and the additional built-in functions unique to Source. + + Provide between 3 and 5 concise comment suggestions, each under 200 words. + + Your output must include only the comment suggestions, separated exclusively by triple pipes ("|||") with no spaces before or after the pipes, and without any additional formatting, bullet points, or extra text. + + Comments and documentation in the code are not necessary for the code, do not penalise based on that, do not suggest to add comments as well. + + Follow the XP scoring guideline provided below in the question prompt, do not be too harsh! + + For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." + """ + + # Get the llm_prompt from the first answer's question + llm_prompt = + answers + |> List.first() + |> Map.get(:question) + |> Map.get(:question) + |> Map.get("llm_prompt") + + # Combine prompts if llm_prompt exists + prompt = + if llm_prompt && llm_prompt != "" do + raw_prompt <> "Additional Instructions:\n\n" <> llm_prompt <> "\n\n" <> answers_json + else + raw_prompt <> "\n" <> answers_json + end - input = %{ - model: @model, - messages: [ - %{role: "system", content: "You are an expert software engineer and educator."}, - %{role: "user", content: prompt} - ], - temperature: 0.5 - } |> Jason.encode!() + input = + %{ + model: @model, + messages: [ + %{role: "system", content: "You are an expert software engineer and educator."}, + %{role: "user", content: prompt} + ], + temperature: 0.5 + } + |> Jason.encode!() headers = [ {"Authorization", "Bearer #{api_key}"}, {"Content-Type", "application/json"} ] - case HTTPoison.post(@openai_api_url, input, headers, timeout: 60_000, recv_timeout: 60_000) do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> case Jason.decode(body) do @@ -261,19 +276,36 @@ defmodule CadetWeb.AICodeAnalysisController do log_comment(submission_id, question_id, prompt, answers_json, response) comments_list = String.split(response, "|||") - filtered_comments = Enum.filter(comments_list, fn comment -> - String.trim(comment) != "" - end) + filtered_comments = + Enum.filter(comments_list, fn comment -> + String.trim(comment) != "" + end) json(conn, %{"comments" => filtered_comments}) {:error, _} -> - log_comment(submission_id, question_id, prompt, answers_json, nil, "Failed to parse response from OpenAI API") + log_comment( + submission_id, + question_id, + prompt, + answers_json, + nil, + "Failed to parse response from OpenAI API" + ) + json(conn, %{"error" => "Failed to parse response from OpenAI API"}) end {:ok, %HTTPoison.Response{status_code: status, body: body}} -> - log_comment(submission_id, question_id, prompt, answers_json, nil, "API request failed with status #{status}") + log_comment( + submission_id, + question_id, + prompt, + answers_json, + nil, + "API request failed with status #{status}" + ) + json(conn, %{"error" => "API request failed with status #{status}: #{body}"}) {:error, %HTTPoison.Error{reason: reason}} -> @@ -285,10 +317,15 @@ defmodule CadetWeb.AICodeAnalysisController do @doc """ Saves the final comment chosen for a submission. """ - def save_final_comment(conn, %{"submissionid" => submission_id, "questionid" => question_id, "comment" => comment}) do + def save_final_comment(conn, %{ + "submissionid" => submission_id, + "questionid" => question_id, + "comment" => comment + }) do case AIComments.update_final_comment(submission_id, question_id, comment) do {:ok, _updated_comment} -> json(conn, %{"status" => "success"}) + {:error, changeset} -> conn |> put_status(:unprocessable_entity) @@ -300,7 +337,11 @@ defmodule CadetWeb.AICodeAnalysisController do Saves the chosen comments for a submission and question. Accepts an array of comments in the request body. """ - def save_chosen_comments(conn, %{"submissionid" => submission_id, "questionid" => question_id, "comments" => comments}) do + def save_chosen_comments(conn, %{ + "submissionid" => submission_id, + "questionid" => question_id, + "comments" => comments + }) do case AIComments.update_chosen_comments(submission_id, question_id, comments) do {:ok, _updated_comment} -> json(conn, %{"status" => "success"}) @@ -375,6 +416,7 @@ defmodule CadetWeb.AICodeAnalysisController do end defp decrypt_llm_api_key(nil), do: nil + defp decrypt_llm_api_key(encrypted_key) do try do # Get the encryption key diff --git a/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs b/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs index 295c83eea..ec3318eda 100644 --- a/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs +++ b/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs @@ -3,7 +3,7 @@ defmodule Cadet.Repo.Migrations.IncreaseLlmApiKeyLength do def change do alter table(:courses) do - modify :llm_api_key, :text + modify(:llm_api_key, :text) end end end diff --git a/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs b/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs index 112f8b4c2..751cf9b7d 100644 --- a/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs +++ b/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs @@ -3,15 +3,15 @@ defmodule Cadet.Repo.Migrations.UpdateCommentChosenToArray do def change do alter table(:ai_comment_logs) do - add :comment_chosen_temp, {:array, :string}, default: [] + add(:comment_chosen_temp, {:array, :string}, default: []) end - execute "UPDATE ai_comment_logs SET comment_chosen_temp = ARRAY[comment_chosen]" + execute("UPDATE ai_comment_logs SET comment_chosen_temp = ARRAY[comment_chosen]") alter table(:ai_comment_logs) do - remove :comment_chosen + remove(:comment_chosen) end - rename table(:ai_comment_logs), :comment_chosen_temp, to: :comment_chosen + rename(table(:ai_comment_logs), :comment_chosen_temp, to: :comment_chosen) end end diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs index 54492c1ee..c17e57174 100644 --- a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -22,7 +22,9 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do # Make the API call response = conn - |> post(Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id)) + |> post( + Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id) + ) |> json_response(200) # Verify database entry @@ -52,7 +54,9 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do # Make the API call that should fail response = conn - |> post(Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id)) + |> post( + Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id) + ) |> json_response(400) # Verify error is logged in database @@ -82,21 +86,26 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do answers_json = ~s({"test": "data"}) response = "Comment 1|||Comment 2|||Comment 3" - {:ok, _comment} = AIComments.create_ai_comment(%{ - submission_id: submission_id, - question_id: question_id, - raw_prompt: raw_prompt, - answers_json: answers_json, - response: response - }) + {:ok, _comment} = + AIComments.create_ai_comment(%{ + submission_id: submission_id, + question_id: question_id, + raw_prompt: raw_prompt, + answers_json: answers_json, + response: response + }) # Now save the final comment final_comment = "This is the chosen final comment" + response = conn - |> post(Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), %{ - comment: final_comment - }) + |> post( + Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), + %{ + comment: final_comment + } + ) |> json_response(200) assert response["status"] == "success" @@ -113,9 +122,12 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do response = conn - |> post(Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), %{ - comment: final_comment - }) + |> post( + Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), + %{ + comment: final_comment + } + ) |> json_response(422) assert response["error"] == "Failed to save final comment" @@ -131,21 +143,26 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do answers_json = ~s({"test": "data"}) response = "Comment 1|||Comment 2|||Comment 3" - {:ok, _comment} = AIComments.create_ai_comment(%{ - submission_id: submission_id, - question_id: question_id, - raw_prompt: raw_prompt, - answers_json: answers_json, - response: response - }) + {:ok, _comment} = + AIComments.create_ai_comment(%{ + submission_id: submission_id, + question_id: question_id, + raw_prompt: raw_prompt, + answers_json: answers_json, + response: response + }) # Now save the chosen comments chosen_comments = ["Comment 1", "Comment 2"] + response = conn - |> post(Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), %{ - comments: chosen_comments - }) + |> post( + Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), + %{ + comments: chosen_comments + } + ) |> json_response(200) assert response["status"] == "success" @@ -162,9 +179,12 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do response = conn - |> post(Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), %{ - comments: chosen_comments - }) + |> post( + Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), + %{ + comments: chosen_comments + } + ) |> json_response(422) assert response["error"] == "Failed to save chosen comments" From 17884fd155a307af80402f5bfb6084aab224eb1b Mon Sep 17 00:00:00 2001 From: Arul Date: Sun, 6 Apr 2025 18:40:48 +0800 Subject: [PATCH 18/77] fix: error handling when calling openai API --- lib/cadet_web/controllers/generate_ai_comments.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 48b81bc6b..9bd86e752 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -306,7 +306,9 @@ defmodule CadetWeb.AICodeAnalysisController do "API request failed with status #{status}" ) - json(conn, %{"error" => "API request failed with status #{status}: #{body}"}) + conn + |> put_status(:internal_server_error) + |> json(%{"error" => "API request failed with status #{status}: #{body}"}) {:error, %HTTPoison.Error{reason: reason}} -> log_comment(submission_id, question_id, prompt, answers_json, nil, reason) From f91cc92891513ef6042616160946fe1a82071297 Mon Sep 17 00:00:00 2001 From: Arul Date: Wed, 9 Apr 2025 12:56:06 +0800 Subject: [PATCH 19/77] fix: credo issues --- lib/cadet/ai_comments.ex | 43 +++++++-------- lib/cadet/ai_comments/ai_comment.ex | 4 ++ .../controllers/generate_ai_comments.ex | 52 +++++++------------ 3 files changed, 42 insertions(+), 57 deletions(-) diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index eaf91a344..58accc768 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -1,4 +1,8 @@ defmodule Cadet.AIComments do + @moduledoc """ + Handles operations related to AI comments, including creation, updates, and retrieval. + """ + import Ecto.Query alias Cadet.Repo alias Cadet.AIComments.AIComment @@ -34,12 +38,13 @@ defmodule Cadet.AIComments do Returns `nil` if no comment exists. """ def get_latest_ai_comment(submission_id, question_id) do - from(c in AIComment, - where: c.submission_id == ^submission_id and c.question_id == ^question_id, - order_by: [desc: c.inserted_at], - limit: 1 + Repo.one( + from(c in AIComment, + where: c.submission_id == ^submission_id and c.question_id == ^question_id, + order_by: [desc: c.inserted_at], + limit: 1 + ) ) - |> Repo.one() end @doc """ @@ -47,17 +52,11 @@ defmodule Cadet.AIComments do Returns the most recent comment entry for that submission/question. """ def update_final_comment(submission_id, question_id, final_comment) do - from(c in AIComment, - where: c.submission_id == ^submission_id and c.question_id == ^question_id, - order_by: [desc: c.inserted_at], - limit: 1 - ) - |> Repo.one() - |> case do - nil -> - {:error, :not_found} + comment = get_latest_ai_comment(submission_id, question_id) - comment -> + case comment do + nil -> {:error, :not_found} + _ -> comment |> AIComment.changeset(%{final_comment: final_comment}) |> Repo.update() @@ -79,17 +78,11 @@ defmodule Cadet.AIComments do Accepts an array of comments and replaces the existing array in the database. """ def update_chosen_comments(submission_id, question_id, new_comments) do - from(c in AIComment, - where: c.submission_id == ^submission_id and c.question_id == ^question_id, - order_by: [desc: c.inserted_at], - limit: 1 - ) - |> Repo.one() - |> case do - nil -> - {:error, :not_found} + comment = get_latest_ai_comment(submission_id, question_id) - comment -> + case comment do + nil -> {:error, :not_found} + _ -> comment |> AIComment.changeset(%{comment_chosen: new_comments}) |> Repo.update() diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex index 0cefe9530..1f1fae478 100644 --- a/lib/cadet/ai_comments/ai_comment.ex +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -1,4 +1,8 @@ defmodule Cadet.AIComments.AIComment do + @moduledoc """ + Defines the schema and changeset for AI comments. + """ + use Ecto.Schema import Ecto.Changeset diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 9bd86e752..1c1503669 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -4,9 +4,7 @@ defmodule CadetWeb.AICodeAnalysisController do require HTTPoison require Logger - alias Cadet.Assessments - alias Cadet.AIComments - alias Cadet.Courses + alias Cadet.{Assessments, AIComments, Courses} @openai_api_url "https://api.openai.com/v1/chat/completions" @model "gpt-4o" @@ -114,12 +112,10 @@ defmodule CadetWeb.AICodeAnalysisController do end) end - def format_answers(json_string) do + defp format_answers(json_string) do {:ok, answers} = Jason.decode(json_string) - answers - |> Enum.map(&format_answer/1) - |> Enum.join("\n\n") + Enum.map_join(answers, "\n\n", &format_answer/1) end defp format_answer(answer) do @@ -148,11 +144,9 @@ defmodule CadetWeb.AICodeAnalysisController do defp format_autograding_results(nil), do: "N/A" defp format_autograding_results(results) when is_list(results) do - results - |> Enum.map(fn result -> + Enum.map_join(results, "; ", fn result -> "Error: #{result["errorMessage"] || "N/A"}, Type: #{result["errorType"] || "N/A"}" end) - |> Enum.join("; ") end defp format_autograding_results(results), do: inspect(results) @@ -420,35 +414,29 @@ defmodule CadetWeb.AICodeAnalysisController do defp decrypt_llm_api_key(nil), do: nil defp decrypt_llm_api_key(encrypted_key) do - try do - # Get the encryption key - secret = Application.get_env(:openai, :encryption_key) - - if is_binary(secret) and byte_size(secret) >= 16 do - # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256 + case Application.get_env(:openai, :encryption_key) do + secret when is_binary(secret) and byte_size(secret) >= 16 -> key = binary_part(secret, 0, min(32, byte_size(secret))) - # Decode the base64 string - decoded = Base.decode64!(encrypted_key) + case Base.decode64(encrypted_key) do + {:ok, decoded} -> + iv = binary_part(decoded, 0, 16) + tag = binary_part(decoded, 16, 16) + ciphertext = binary_part(decoded, 32, byte_size(decoded) - 32) - # Extract IV, tag and ciphertext - iv = binary_part(decoded, 0, 16) - tag = binary_part(decoded, 16, 16) - ciphertext = binary_part(decoded, 32, byte_size(decoded) - 32) + case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do + plain_text when is_binary(plain_text) -> plain_text + _ -> nil + end - # Decrypt - case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do - plain_text when is_binary(plain_text) -> plain_text - _ -> nil + _ -> + Logger.error("Failed to decode encrypted key") + nil end - else + + _ -> Logger.error("Encryption key not configured properly") nil - end - rescue - e -> - Logger.error("Error decrypting LLM API key: #{inspect(e)}") - nil end end end From 81e5bf74d32dd4d6e2f95e1d9e0f2b1d558151b5 Mon Sep 17 00:00:00 2001 From: Arul Date: Wed, 9 Apr 2025 13:01:05 +0800 Subject: [PATCH 20/77] formatting --- lib/cadet/ai_comments.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index 58accc768..76b059601 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -55,7 +55,9 @@ defmodule Cadet.AIComments do comment = get_latest_ai_comment(submission_id, question_id) case comment do - nil -> {:error, :not_found} + nil -> + {:error, :not_found} + _ -> comment |> AIComment.changeset(%{final_comment: final_comment}) @@ -81,7 +83,9 @@ defmodule Cadet.AIComments do comment = get_latest_ai_comment(submission_id, question_id) case comment do - nil -> {:error, :not_found} + nil -> + {:error, :not_found} + _ -> comment |> AIComment.changeset(%{comment_chosen: new_comments}) From 1ec67d7ba502934cc745535a2220f40baf04bf85 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Sat, 27 Sep 2025 12:31:59 +0800 Subject: [PATCH 21/77] Address some comments --- lib/cadet/ai_comments.ex | 18 ++++++++++++++---- .../controllers/generate_ai_comments.ex | 17 +++++++++++------ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index 76b059601..234fed553 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -19,7 +19,12 @@ defmodule Cadet.AIComments do @doc """ Gets an AI comment by ID. """ - def get_ai_comment!(id), do: Repo.get!(AIComment, id) + def get_ai_comment(id) do + case Repo.get(AIComment, id) do + nil -> {:error, :not_found} + comment -> {:ok, comment} + end + end @doc """ Retrieves an AI comment for a specific submission and question. @@ -70,9 +75,14 @@ defmodule Cadet.AIComments do """ def update_ai_comment(id, attrs) do id - |> get_ai_comment!() - |> AIComment.changeset(attrs) - |> Repo.update() + |> get_ai_comment() + |> case do + {:error, :not_found} -> {:error, :not_found} + {:ok, comment} -> + comment + |> AIComment.changeset(attrs) + |> Repo.update() + end end @doc """ diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 1c1503669..21a1c05f1 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -12,7 +12,7 @@ defmodule CadetWeb.AICodeAnalysisController do @default_llm_grading false # For logging outputs to both database and file - defp log_comment(submission_id, question_id, raw_prompt, answers_json, response, error \\ nil) do + defp save_comment(submission_id, question_id, raw_prompt, answers_json, response, error \\ nil) do # Log to database attrs = %{ submission_id: submission_id, @@ -42,6 +42,10 @@ defmodule CadetWeb.AICodeAnalysisController do updated_attrs = Map.merge(Map.from_struct(existing_comment), attrs) case AIComments.update_ai_comment(existing_comment.id, updated_attrs) do + {:error, :not_found} -> + Logger.error("AI comment to update not found in database") + {:error, :not_found} + {:ok, updated_comment} -> {:ok, updated_comment} @@ -65,7 +69,6 @@ defmodule CadetWeb.AICodeAnalysisController do case Courses.get_course_config(course_id) do {:ok, course} -> if course.enable_llm_grading || @default_llm_grading do - Logger.info("LLM Api key: #{course.llm_api_key}") # Get API key from course config or fall back to environment variable decrypted_api_key = decrypt_llm_api_key(course.llm_api_key) api_key = decrypted_api_key || Application.get_env(:openai, :api_key) @@ -155,6 +158,8 @@ defmodule CadetWeb.AICodeAnalysisController do answers_json = answers |> Enum.map(fn answer -> + IO.inspect(answer, label: "Answer:") + question_data = if answer.question do %{ @@ -267,7 +272,7 @@ defmodule CadetWeb.AICodeAnalysisController do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> case Jason.decode(body) do {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> - log_comment(submission_id, question_id, prompt, answers_json, response) + save_comment(submission_id, question_id, prompt, answers_json, response) comments_list = String.split(response, "|||") filtered_comments = @@ -278,7 +283,7 @@ defmodule CadetWeb.AICodeAnalysisController do json(conn, %{"comments" => filtered_comments}) {:error, _} -> - log_comment( + save_comment( submission_id, question_id, prompt, @@ -291,7 +296,7 @@ defmodule CadetWeb.AICodeAnalysisController do end {:ok, %HTTPoison.Response{status_code: status, body: body}} -> - log_comment( + save_comment( submission_id, question_id, prompt, @@ -305,7 +310,7 @@ defmodule CadetWeb.AICodeAnalysisController do |> json(%{"error" => "API request failed with status #{status}: #{body}"}) {:error, %HTTPoison.Error{reason: reason}} -> - log_comment(submission_id, question_id, prompt, answers_json, nil, reason) + save_comment(submission_id, question_id, prompt, answers_json, nil, reason) json(conn, %{"error" => "HTTP request error: #{inspect(reason)}"}) end end From 4f2af5da053f2083c6061f73a0263116d921d7d7 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Sat, 27 Sep 2025 12:35:47 +0800 Subject: [PATCH 22/77] Fix formatting --- lib/cadet/ai_comments.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index 234fed553..8e5ce62ac 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -77,7 +77,9 @@ defmodule Cadet.AIComments do id |> get_ai_comment() |> case do - {:error, :not_found} -> {:error, :not_found} + {:error, :not_found} -> + {:error, :not_found} + {:ok, comment} -> comment |> AIComment.changeset(attrs) From ec67aa37f2a2b6c40c5789e2f1eee014653612bd Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Sat, 27 Sep 2025 12:36:35 +0800 Subject: [PATCH 23/77] rm IO.inspect --- lib/cadet_web/controllers/generate_ai_comments.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 21a1c05f1..becddf929 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -158,7 +158,6 @@ defmodule CadetWeb.AICodeAnalysisController do answers_json = answers |> Enum.map(fn answer -> - IO.inspect(answer, label: "Answer:") question_data = if answer.question do From 11ff272d36debca98b25028b7d1c5d718bd48137 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Sat, 27 Sep 2025 12:36:55 +0800 Subject: [PATCH 24/77] a --- lib/cadet_web/controllers/generate_ai_comments.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index becddf929..2c9e774f1 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -158,7 +158,6 @@ defmodule CadetWeb.AICodeAnalysisController do answers_json = answers |> Enum.map(fn answer -> - question_data = if answer.question do %{ From 1a77f67bd5771c0c44d5006a1cd6d29297b53d6a Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Sat, 27 Sep 2025 12:50:00 +0800 Subject: [PATCH 25/77] Use case instead of if --- lib/cadet/assessments/assessments.ex | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index ac0018e90..2ecb3060e 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -2310,10 +2310,9 @@ defmodule Cadet.Assessments do ) answer_query = - if is_nil(question_id) do - base_query - else - base_query |> where(question_id: ^question_id) + case question_id do + nil -> base_query + _ -> base_query |> where(question_id: ^question_id) end answers = From 02922e9d79886b02c4aaa3cf43d7792d2fdfd1a0 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Sat, 27 Sep 2025 19:06:32 +0800 Subject: [PATCH 26/77] Streamlines generate_ai_comments to only send the selected question and its relevant info + use the correct llm_prompt --- .../controllers/generate_ai_comments.ex | 305 ++++++++---------- 1 file changed, 141 insertions(+), 164 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 2c9e774f1..bd73b60fa 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -7,18 +7,18 @@ defmodule CadetWeb.AICodeAnalysisController do alias Cadet.{Assessments, AIComments, Courses} @openai_api_url "https://api.openai.com/v1/chat/completions" - @model "gpt-4o" + @model "gpt-5-mini" # To set whether LLM grading is enabled across Source Academy @default_llm_grading false # For logging outputs to both database and file - defp save_comment(submission_id, question_id, raw_prompt, answers_json, response, error \\ nil) do + defp save_comment(submission_id, question_id, raw_prompt, answer, response, error \\ nil) do # Log to database attrs = %{ submission_id: submission_id, question_id: question_id, raw_prompt: raw_prompt, - answers_json: answers_json, + answer: answer, response: response, error: error, inserted_at: NaiveDateTime.utc_now() @@ -66,6 +66,8 @@ defmodule CadetWeb.AICodeAnalysisController do }) when is_ecto_id(submission_id) do # Check if LLM grading is enabled for this course (default to @default_llm_grading if nil) + question_id = String.to_integer(question_id) + case Courses.get_course_config(course_id) do {:ok, course} -> if course.enable_llm_grading || @default_llm_grading do @@ -80,7 +82,15 @@ defmodule CadetWeb.AICodeAnalysisController do else case Assessments.get_answers_in_submission(submission_id, question_id) do {:ok, {answers, _assessment}} -> - analyze_code(conn, answers, submission_id, question_id, api_key) + case answers do + [] -> + conn + |> put_status(:not_found) + |> json(%{"error" => "No answer found for the given submission and question_id"}) + _ -> + # Get head of answers (should only be one answer for given submission and question) + analyze_code(conn, hd(answers), submission_id, question_id, api_key) + end {:error, {status, message}} -> conn @@ -115,32 +125,26 @@ defmodule CadetWeb.AICodeAnalysisController do end) end - defp format_answers(json_string) do - {:ok, answers} = Jason.decode(json_string) - - Enum.map_join(answers, "\n\n", &format_answer/1) - end - defp format_answer(answer) do """ - **Question ID: #{answer["question"]["id"] || "N/A"}** + **Question ID: #{answer.question.id || "N/A"}** **Question:** - #{answer["question"]["content"] || "N/A"} + #{answer.question.question["content"] || "N/A"} - **Solution:** + **Model Solution:** ``` - #{answer["question"]["solution"] || "N/A"} + #{answer.question.question["solution"] || "N/A"} ``` - **Answer:** + **Student Answer:** ``` - #{answer["answer"]["code"] || "N/A"} + #{answer.answer["code"] || "N/A"} ``` - **Autograding Status:** #{answer["autograding_status"] || "N/A"} - **Autograding Results:** #{format_autograding_results(answer["autograding_results"])} - **Comments:** #{answer["comments"] || "None"} + **Autograding Status:** #{answer.autograding_status || "N/A"} + **Autograding Results:** #{format_autograding_results(answer.autograding_results)} + **Comments:** #{answer.comments || "None"} """ end @@ -154,164 +158,137 @@ defmodule CadetWeb.AICodeAnalysisController do defp format_autograding_results(results), do: inspect(results) - defp analyze_code(conn, answers, submission_id, question_id, api_key) do - answers_json = - answers - |> Enum.map(fn answer -> - question_data = - if answer.question do - %{ - id: answer.question_id, - content: Map.get(answer.question.question, "content"), - solution: Map.get(answer.question.question, "solution"), - llm_prompt: Map.get(answer.question.question, "llm_prompt") - } + defp analyze_code(conn, answer, submission_id, question_id, api_key) do + + raw_prompt = """ + The code below is written in Source, a variant of JavaScript that comes with a rich set of built-in constants and functions. Below is a summary of some key built-in entities available in Source: + + Constants: + - Infinity: The special number value representing infinity. + - NaN: The special number value for "not a number." + - undefined: The special value for an undefined variable. + - math_PI: The constant π (approximately 3.14159). + - math_E: Euler's number (approximately 2.71828). + + Functions: + - __access_export__(exports, lookup_name): Searches for a name in an exports data structure. + - accumulate(f, initial, xs): Reduces a list by applying a binary function from right-to-left. + - append(xs, ys): Appends list ys to the end of list xs. + - char_at(s, i): Returns the character at index i of string s. + - display(v, s): Displays value v (optionally preceded by string s) in the console. + - filter(pred, xs): Returns a new list with elements of xs that satisfy the predicate pred. + - for_each(f, xs): Applies function f to each element of the list xs. + - get_time(): Returns the current time in milliseconds. + - is_list(xs): Checks whether xs is a proper list. + - length(xs): Returns the number of elements in list xs. + - list(...): Constructs a list from the provided values. + - map(f, xs): Applies function f to each element of list xs. + - math_abs(x): Returns the absolute value of x. + - math_ceil(x): Rounds x up to the nearest integer. + - math_floor(x): Rounds x down to the nearest integer. + - pair(x, y): A primitive function that makes a pair whose head (first component) is x and whose tail (second component) is y. + - head(xs): Returns the first element of pair xs. + - tail(xs): Returns the second element of pair xs. + - math_random(): Returns a random number between 0 (inclusive) and 1 (exclusive). + + (For a full list of built-in functions and constants, refer to the Source documentation.) + + Analyze the following submitted "Student Answer" (ONLY) against the given information and provide detailed feedback on correctness, readability, efficiency, and possible improvements. Your evaluation should consider both standard JavaScript features and the additional built-in functions unique to Source. + + Provide between 3 and 5 concise comment suggestions, each under 200 words. + + Your output must include only the comment suggestions, separated exclusively by triple pipes ("|||") with no spaces before or after the pipes, and without any additional formatting, bullet points, or extra text. + + Comments and documentation in the code are not necessary for the code, do not penalise based on that, do not suggest to add comments as well. + + Follow the XP scoring guideline provided below in the question prompt, do not be too harsh! + + For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." + """ + + IO.inspect(answer, label: "Answer being analyzed") + formatted_answer = + format_answer(answer) + |> Jason.encode!() + + IO.inspect("Formatted answer: #{formatted_answer}", label: "Formatted Answer") + llm_prompt = answer.question.question["llm_prompt"] # "llm_prompt" is a string key, so we can't use the (.) operator directly. optional + + # Combine prompts if llm_prompt exists + prompt = + if llm_prompt && llm_prompt != "" do + raw_prompt <> "Additional Instructions:\n\n" <> llm_prompt <> "\n\n" <> formatted_answer else - %{ - id: nil, - content: nil, - llm_prompt: nil - } + raw_prompt <> "\n" <> formatted_answer end - answer - |> Map.from_struct() - |> Map.take([ - :id, - :comments, - :autograding_status, - :autograding_results, - :answer - ]) - |> Map.put(:question, question_data) - end) - |> Jason.encode!() - |> format_answers() - - raw_prompt = """ - The code below is written in Source, a variant of JavaScript that comes with a rich set of built-in constants and functions. Below is a summary of some key built-in entities available in Source: - - Constants: - - Infinity: The special number value representing infinity. - - NaN: The special number value for "not a number." - - undefined: The special value for an undefined variable. - - math_PI: The constant π (approximately 3.14159). - - math_E: Euler's number (approximately 2.71828). - - Functions: - - __access_export__(exports, lookup_name): Searches for a name in an exports data structure. - - accumulate(f, initial, xs): Reduces a list by applying a binary function from right-to-left. - - append(xs, ys): Appends list ys to the end of list xs. - - char_at(s, i): Returns the character at index i of string s. - - display(v, s): Displays value v (optionally preceded by string s) in the console. - - filter(pred, xs): Returns a new list with elements of xs that satisfy the predicate pred. - - for_each(f, xs): Applies function f to each element of the list xs. - - get_time(): Returns the current time in milliseconds. - - is_list(xs): Checks whether xs is a proper list. - - length(xs): Returns the number of elements in list xs. - - list(...): Constructs a list from the provided values. - - map(f, xs): Applies function f to each element of list xs. - - math_abs(x): Returns the absolute value of x. - - math_ceil(x): Rounds x up to the nearest integer. - - math_floor(x): Rounds x down to the nearest integer. - - pair(x, y): A primitive function that makes a pair whose head (first component) is x and whose tail (second component) is y. - - head(xs): Returns the first element of pair xs. - - tail(xs): Returns the second element of pair xs. - - math_random(): Returns a random number between 0 (inclusive) and 1 (exclusive). - - (For a full list of built-in functions and constants, refer to the Source documentation.) - - Analyze the following submitted answers and provide detailed feedback on correctness, readability, efficiency, and possible improvements. Your evaluation should consider both standard JavaScript features and the additional built-in functions unique to Source. - - Provide between 3 and 5 concise comment suggestions, each under 200 words. - - Your output must include only the comment suggestions, separated exclusively by triple pipes ("|||") with no spaces before or after the pipes, and without any additional formatting, bullet points, or extra text. - - Comments and documentation in the code are not necessary for the code, do not penalise based on that, do not suggest to add comments as well. - - Follow the XP scoring guideline provided below in the question prompt, do not be too harsh! - - For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." - """ - - # Get the llm_prompt from the first answer's question - llm_prompt = - answers - |> List.first() - |> Map.get(:question) - |> Map.get(:question) - |> Map.get("llm_prompt") - - # Combine prompts if llm_prompt exists - prompt = - if llm_prompt && llm_prompt != "" do - raw_prompt <> "Additional Instructions:\n\n" <> llm_prompt <> "\n\n" <> answers_json - else - raw_prompt <> "\n" <> answers_json - end - - input = - %{ - model: @model, - messages: [ - %{role: "system", content: "You are an expert software engineer and educator."}, - %{role: "user", content: prompt} - ], - temperature: 0.5 - } - |> Jason.encode!() - - headers = [ - {"Authorization", "Bearer #{api_key}"}, - {"Content-Type", "application/json"} - ] - - case HTTPoison.post(@openai_api_url, input, headers, timeout: 60_000, recv_timeout: 60_000) do - {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> - case Jason.decode(body) do - {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> - save_comment(submission_id, question_id, prompt, answers_json, response) - comments_list = String.split(response, "|||") - - filtered_comments = - Enum.filter(comments_list, fn comment -> - String.trim(comment) != "" - end) - - json(conn, %{"comments" => filtered_comments}) + input = + %{ + model: @model, + messages: [ + %{role: "system", content: "You are an expert software engineer and educator."}, + %{role: "user", content: prompt} + ], + } + |> Jason.encode!() + + headers = [ + {"Authorization", "Bearer #{api_key}"}, + {"Content-Type", "application/json"} + ] + + IO.inspect(input, label: "Input to OpenAI API") + case HTTPoison.post(@openai_api_url, input, headers, + timeout: 60_000, + recv_timeout: 60_000 + ) do + {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> + case Jason.decode(body) do + {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> + IO.inspect(response, label: "Response from OpenAI API") + save_comment(submission_id, question_id, prompt, formatted_answer, response) + comments_list = String.split(response, "|||") + + filtered_comments = + Enum.filter(comments_list, fn comment -> + String.trim(comment) != "" + end) + + json(conn, %{"comments" => filtered_comments}) + + {:error, _} -> + save_comment( + submission_id, + question_id, + prompt, + formatted_answer, + nil, + "Failed to parse response from OpenAI API" + ) + + json(conn, %{"error" => "Failed to parse response from OpenAI API"}) + end - {:error, _} -> + {:ok, %HTTPoison.Response{status_code: status, body: body}} -> + IO.inspect(body, label: "Error response from OpenAI API") save_comment( submission_id, question_id, prompt, - answers_json, + formatted_answer, nil, - "Failed to parse response from OpenAI API" + "API request failed with status #{status}" ) - json(conn, %{"error" => "Failed to parse response from OpenAI API"}) - end - - {:ok, %HTTPoison.Response{status_code: status, body: body}} -> - save_comment( - submission_id, - question_id, - prompt, - answers_json, - nil, - "API request failed with status #{status}" - ) - - conn - |> put_status(:internal_server_error) - |> json(%{"error" => "API request failed with status #{status}: #{body}"}) + conn + |> put_status(:internal_server_error) + |> json(%{"error" => "API request failed with status #{status}: #{body}"}) - {:error, %HTTPoison.Error{reason: reason}} -> - save_comment(submission_id, question_id, prompt, answers_json, nil, reason) - json(conn, %{"error" => "HTTP request error: #{inspect(reason)}"}) + {:error, %HTTPoison.Error{reason: reason}} -> + save_comment(submission_id, question_id, prompt, formatted_answer, nil, reason) + json(conn, %{"error" => "HTTP request error: #{inspect(reason)}"}) + end end - end @doc """ Saves the final comment chosen for a submission. From f068aa961264c0c8cb16cee1c1d9e506f71a9370 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Sat, 27 Sep 2025 19:12:15 +0800 Subject: [PATCH 27/77] Remove unncessary field --- lib/cadet_web/controllers/generate_ai_comments.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index bd73b60fa..b5d33581d 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -12,16 +12,15 @@ defmodule CadetWeb.AICodeAnalysisController do @default_llm_grading false # For logging outputs to both database and file - defp save_comment(submission_id, question_id, raw_prompt, answer, response, error \\ nil) do + defp save_comment(submission_id, question_id, raw_prompt, answers_json, response, error \\ nil) do # Log to database attrs = %{ submission_id: submission_id, question_id: question_id, raw_prompt: raw_prompt, - answer: answer, + answers_json: answers_json, response: response, error: error, - inserted_at: NaiveDateTime.utc_now() } # Check if a comment already exists for the given submission_id and question_id From 5f3ad2ca84b52add63ba0ed291e588ef05c3c272 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Sat, 27 Sep 2025 19:13:09 +0800 Subject: [PATCH 28/77] default: false for llm_grading --- lib/cadet/courses/course.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index cf6170cfb..8771a0ddc 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -30,7 +30,7 @@ defmodule Cadet.Courses.Course do field(:enable_achievements, :boolean, default: true) field(:enable_sourcecast, :boolean, default: true) field(:enable_stories, :boolean, default: false) - field(:enable_llm_grading, :boolean) + field(:enable_llm_grading, :boolean, default: false) field(:llm_api_key, :string) field(:source_chapter, :integer) field(:source_variant, :string) From 34d326c3d70e89c918da8e49e85b187577d13401 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Sun, 28 Sep 2025 18:24:36 +0800 Subject: [PATCH 29/77] Add proper linking between ai_comments table and submissions. Return it to submission retrieval as well --- lib/cadet/ai_comments/ai_comment.ex | 7 +++++-- lib/cadet/assessments/answer.ex | 1 + lib/cadet/assessments/assessments.ex | 10 ++++++---- lib/cadet/assessments/submission.ex | 1 + lib/cadet_web/admin_views/admin_grading_view.ex | 11 +++++++++++ .../migrations/20250220103623_create_ai_comments.exs | 4 ++-- 6 files changed, 26 insertions(+), 8 deletions(-) diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex index 1f1fae478..3dad339b7 100644 --- a/lib/cadet/ai_comments/ai_comment.ex +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -7,8 +7,6 @@ defmodule Cadet.AIComments.AIComment do import Ecto.Changeset schema "ai_comment_logs" do - field(:submission_id, :integer) - field(:question_id, :integer) field(:raw_prompt, :string) field(:answers_json, :string) field(:response, :string) @@ -16,6 +14,9 @@ defmodule Cadet.AIComments.AIComment do field(:comment_chosen, {:array, :string}) field(:final_comment, :string) + belongs_to(:submission, Cadet.Assessments.Submission) + belongs_to(:question, Cadet.Assessments.Question) + timestamps() end @@ -32,5 +33,7 @@ defmodule Cadet.AIComments.AIComment do :final_comment ]) |> validate_required([:submission_id, :question_id, :raw_prompt, :answers_json]) + |> foreign_key_constraint(:submission_id) + |> foreign_key_constraint(:question_id) end end diff --git a/lib/cadet/assessments/answer.ex b/lib/cadet/assessments/answer.ex index a6794117b..cab783a7d 100644 --- a/lib/cadet/assessments/answer.ex +++ b/lib/cadet/assessments/answer.ex @@ -29,6 +29,7 @@ defmodule Cadet.Assessments.Answer do belongs_to(:grader, CourseRegistration) belongs_to(:submission, Submission) belongs_to(:question, Question) + has_many(:ai_comments, through: [:submission, :ai_comments]) timestamps() end diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 2ecb3060e..9ed7abb3a 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -2290,19 +2290,21 @@ defmodule Cadet.Assessments do base_query = Answer |> where(submission_id: ^id) - |> join(:inner, [a], q in assoc(a, :question)) + |> join(:inner, [a], q in assoc(a, :question)) # [a] are bindings (in SQL it is similar to FROM answers "AS a"), this line's alias is INNER JOIN ... "AS q" |> join(:inner, [_, q], ast in assoc(q, :assessment)) |> join(:inner, [..., ast], ac in assoc(ast, :config)) |> join(:left, [a, ...], g in assoc(a, :grader)) |> join(:left, [_, ..., g], gu in assoc(g, :user)) |> join(:inner, [a, ...], s in assoc(a, :submission)) - |> join(:left, [_, ..., s], st in assoc(s, :student)) + |> join(:left, [..., s], ai in assoc(s, :ai_comments)) + |> join(:left, [_, ..., s, _], st in assoc(s, :student)) |> join(:left, [..., st], u in assoc(st, :user)) - |> join(:left, [..., s, _, _], t in assoc(s, :team)) + |> join(:left, [..., s, _, _, _], t in assoc(s, :team)) |> join(:left, [..., t], tm in assoc(t, :team_members)) |> join(:left, [..., tm], tms in assoc(tm, :student)) |> join(:left, [..., tms], tmu in assoc(tms, :user)) - |> preload([_, q, ast, ac, g, gu, s, st, u, t, tm, tms, tmu], + |> preload([_, q, ast, ac, g, gu, s, ai, st, u, t, tm, tms, tmu], + ai_comments: ai, question: {q, assessment: {ast, config: ac}}, grader: {g, user: gu}, submission: diff --git a/lib/cadet/assessments/submission.ex b/lib/cadet/assessments/submission.ex index 658f79f1f..5ef3b0574 100644 --- a/lib/cadet/assessments/submission.ex +++ b/lib/cadet/assessments/submission.ex @@ -25,6 +25,7 @@ defmodule Cadet.Assessments.Submission do belongs_to(:team, Team) belongs_to(:unsubmitted_by, CourseRegistration) has_many(:answers, Answer, on_delete: :delete_all) + has_many(:ai_comments, Cadet.AIComments.AIComment, on_delete: :delete_all) timestamps() end diff --git a/lib/cadet_web/admin_views/admin_grading_view.ex b/lib/cadet_web/admin_views/admin_grading_view.ex index 313a27f05..87184af95 100644 --- a/lib/cadet_web/admin_views/admin_grading_view.ex +++ b/lib/cadet_web/admin_views/admin_grading_view.ex @@ -149,6 +149,7 @@ defmodule CadetWeb.AdminGradingView do def render("grading_info.json", %{answer: answer}) do transform_map_for_view(answer, %{ + ai_comments: &extract_ai_comments_per_answer(&1.question_id, &1.ai_comments), student: &extract_student_data(&1.submission.student), team: &extract_team_data(&1.submission.team), question: &build_grading_question/1, @@ -161,6 +162,16 @@ defmodule CadetWeb.AdminGradingView do %{cols: cols, rows: summary} end + defp extract_ai_comments_per_answer(question_id, ai_comments) do + matching_comment = ai_comments + |> Enum.find(&(&1.question_id == question_id)) # Equivalent to fn comment -> comment.question_id == question_id end + + case matching_comment do + nil -> nil + comment -> %{response: matching_comment.response, insertedAt: matching_comment.inserted_at} + end + end + defp extract_student_data(nil), do: %{} defp extract_student_data(student) do diff --git a/priv/repo/migrations/20250220103623_create_ai_comments.exs b/priv/repo/migrations/20250220103623_create_ai_comments.exs index 6338dccd8..c13f13114 100644 --- a/priv/repo/migrations/20250220103623_create_ai_comments.exs +++ b/priv/repo/migrations/20250220103623_create_ai_comments.exs @@ -3,8 +3,8 @@ defmodule Cadet.Repo.Migrations.CreateAiCommentLogs do def change do create table(:ai_comment_logs) do - add(:submission_id, :integer, null: false) - add(:question_id, :integer, null: false) + add(:submission_id, references(:submissions, on_delete: :delete_all), null: false) + add(:question_id, references(:questions, on_delete: :delete_all), null: false) add(:raw_prompt, :text, null: false) add(:answers_json, :text, null: false) add(:response, :text) From 6ff28648f7d9f05d55c0a288668d167c4dd4a97f Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Tue, 30 Sep 2025 14:28:17 +0800 Subject: [PATCH 30/77] Resolve some migration comments --- ...0240320000001_add_llm_api_key_to_courses.exs | 2 +- .../20250220103623_create_ai_comments.exs | 2 +- ...250402030934_increase_llm_api_key_length.exs | 9 --------- ...406053008_update_comment_chosen_to_array.exs | 17 ----------------- 4 files changed, 2 insertions(+), 28 deletions(-) delete mode 100644 priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs delete mode 100644 priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs diff --git a/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs index e752889b7..63041e1ca 100644 --- a/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs +++ b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs @@ -3,7 +3,7 @@ defmodule Cadet.Repo.Migrations.AddLlmApiKeyToCourses do def up do alter table(:courses) do - add(:llm_api_key, :string, null: true) + add(:llm_api_key, :text, null: true) end end diff --git a/priv/repo/migrations/20250220103623_create_ai_comments.exs b/priv/repo/migrations/20250220103623_create_ai_comments.exs index c13f13114..53ee4f532 100644 --- a/priv/repo/migrations/20250220103623_create_ai_comments.exs +++ b/priv/repo/migrations/20250220103623_create_ai_comments.exs @@ -9,7 +9,7 @@ defmodule Cadet.Repo.Migrations.CreateAiCommentLogs do add(:answers_json, :text, null: false) add(:response, :text) add(:error, :text) - add(:comment_chosen, :text) + add(:comment_chosen, {:array, :text}) add(:final_comment, :text) timestamps() end diff --git a/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs b/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs deleted file mode 100644 index ec3318eda..000000000 --- a/priv/repo/migrations/20250402030934_increase_llm_api_key_length.exs +++ /dev/null @@ -1,9 +0,0 @@ -defmodule Cadet.Repo.Migrations.IncreaseLlmApiKeyLength do - use Ecto.Migration - - def change do - alter table(:courses) do - modify(:llm_api_key, :text) - end - end -end diff --git a/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs b/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs deleted file mode 100644 index 751cf9b7d..000000000 --- a/priv/repo/migrations/20250406053008_update_comment_chosen_to_array.exs +++ /dev/null @@ -1,17 +0,0 @@ -defmodule Cadet.Repo.Migrations.UpdateCommentChosenToArray do - use Ecto.Migration - - def change do - alter table(:ai_comment_logs) do - add(:comment_chosen_temp, {:array, :string}, default: []) - end - - execute("UPDATE ai_comment_logs SET comment_chosen_temp = ARRAY[comment_chosen]") - - alter table(:ai_comment_logs) do - remove(:comment_chosen) - end - - rename(table(:ai_comment_logs), :comment_chosen_temp, to: :comment_chosen) - end -end From 08548229f487a05bcd4bb958950e789437f7e537 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Tue, 30 Sep 2025 14:49:29 +0800 Subject: [PATCH 31/77] Add llm_model and llm_api_url to the DB + schema --- lib/cadet/courses/course.ex | 2 + .../controllers/generate_ai_comments.ex | 39 +++++++++++-------- ...40320000001_add_llm_api_key_to_courses.exs | 4 ++ 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index 8771a0ddc..c744ad202 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -32,6 +32,8 @@ defmodule Cadet.Courses.Course do field(:enable_stories, :boolean, default: false) field(:enable_llm_grading, :boolean, default: false) field(:llm_api_key, :string) + field(:llm_model, :string, default: "gpt-5-mini") + field(:llm_api_url, :string, default: "https://api.openai.com/v1/chat/completions") field(:source_chapter, :integer) field(:source_variant, :string) field(:module_help_text, :string) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index b5d33581d..eb9c7dbc8 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -6,11 +6,6 @@ defmodule CadetWeb.AICodeAnalysisController do alias Cadet.{Assessments, AIComments, Courses} - @openai_api_url "https://api.openai.com/v1/chat/completions" - @model "gpt-5-mini" - # To set whether LLM grading is enabled across Source Academy - @default_llm_grading false - # For logging outputs to both database and file defp save_comment(submission_id, question_id, raw_prompt, answers_json, response, error \\ nil) do # Log to database @@ -64,22 +59,31 @@ defmodule CadetWeb.AICodeAnalysisController do "course_id" => course_id }) when is_ecto_id(submission_id) do - # Check if LLM grading is enabled for this course (default to @default_llm_grading if nil) + # Check if LLM grading is enabled for this course question_id = String.to_integer(question_id) case Courses.get_course_config(course_id) do {:ok, course} -> - if course.enable_llm_grading || @default_llm_grading do + if course.enable_llm_grading do # Get API key from course config or fall back to environment variable decrypted_api_key = decrypt_llm_api_key(course.llm_api_key) api_key = decrypted_api_key || Application.get_env(:openai, :api_key) - if is_nil(api_key) do - conn - |> put_status(:internal_server_error) - |> json(%{"error" => "No OpenAI API key configured"}) - else - case Assessments.get_answers_in_submission(submission_id, question_id) do + cond do + is_nil(api_key) -> + conn + |> put_status(:internal_server_error) + |> json(%{"error" => "No OpenAI API key configured"}) + is_nil(course.llm_model) or course.llm_model == "" -> + conn + |> put_status(:internal_server_error) + |> json(%{"error" => "No LLM model configured for this course"}) + is_nil(course.llm_api_url) or course.llm_api_url == "" -> + conn + |> put_status(:internal_server_error) + |> json(%{"error" => "No LLM API URL configured for this course"}) + true -> + case Assessments.get_answers_in_submission(submission_id, question_id) do {:ok, {answers, _assessment}} -> case answers do [] -> @@ -88,7 +92,7 @@ defmodule CadetWeb.AICodeAnalysisController do |> json(%{"error" => "No answer found for the given submission and question_id"}) _ -> # Get head of answers (should only be one answer for given submission and question) - analyze_code(conn, hd(answers), submission_id, question_id, api_key) + analyze_code(conn, hd(answers), submission_id, question_id, api_key, course.llm_model, course.llm_api_url) end {:error, {status, message}} -> @@ -96,6 +100,7 @@ defmodule CadetWeb.AICodeAnalysisController do |> put_status(status) |> text(message) end + end else conn @@ -157,7 +162,7 @@ defmodule CadetWeb.AICodeAnalysisController do defp format_autograding_results(results), do: inspect(results) - defp analyze_code(conn, answer, submission_id, question_id, api_key) do + defp analyze_code(conn, answer, submission_id, question_id, api_key, llm_model, llm_api_url) do raw_prompt = """ The code below is written in Source, a variant of JavaScript that comes with a rich set of built-in constants and functions. Below is a summary of some key built-in entities available in Source: @@ -223,7 +228,7 @@ defmodule CadetWeb.AICodeAnalysisController do input = %{ - model: @model, + model: llm_model, messages: [ %{role: "system", content: "You are an expert software engineer and educator."}, %{role: "user", content: prompt} @@ -237,7 +242,7 @@ defmodule CadetWeb.AICodeAnalysisController do ] IO.inspect(input, label: "Input to OpenAI API") - case HTTPoison.post(@openai_api_url, input, headers, + case HTTPoison.post(llm_api_url, input, headers, timeout: 60_000, recv_timeout: 60_000 ) do diff --git a/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs index 63041e1ca..c2a0b89f8 100644 --- a/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs +++ b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs @@ -4,12 +4,16 @@ defmodule Cadet.Repo.Migrations.AddLlmApiKeyToCourses do def up do alter table(:courses) do add(:llm_api_key, :text, null: true) + add(:llm_model, :text, null: false, default: "gpt-5-mini") + add(:llm_api_url, :text, null: false, default: "https://api.openai.com/v1/chat/completions") end end def down do alter table(:courses) do remove(:llm_api_key) + remove(:llm_model) + remove(:llm_api_url) end end end From f0ccaf60fe522fa2725bb59bbcb26932e99a780c Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Tue, 30 Sep 2025 16:48:25 +0800 Subject: [PATCH 32/77] Moves api key, api url, llm model and course prompt to course level --- lib/cadet/courses/course.ex | 12 +- .../controllers/generate_ai_comments.ex | 105 ++++++------------ lib/cadet_web/views/courses_view.ex | 3 + ...40320000001_add_llm_api_key_to_courses.exs | 7 +- 4 files changed, 51 insertions(+), 76 deletions(-) diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index c744ad202..a84d58cfd 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -16,6 +16,9 @@ defmodule Cadet.Courses.Course do enable_stories: boolean(), enable_llm_grading: boolean(), llm_api_key: String.t() | nil, + llm_model: String.t() | nil, + llm_api_url: String.t() | nil, + llm_course_level_prompt: String.t() | nil, source_chapter: integer(), source_variant: String.t(), module_help_text: String.t(), @@ -31,9 +34,10 @@ defmodule Cadet.Courses.Course do field(:enable_sourcecast, :boolean, default: true) field(:enable_stories, :boolean, default: false) field(:enable_llm_grading, :boolean, default: false) - field(:llm_api_key, :string) - field(:llm_model, :string, default: "gpt-5-mini") - field(:llm_api_url, :string, default: "https://api.openai.com/v1/chat/completions") + field(:llm_api_key, :string, default: nil) + field(:llm_model, :string, default: nil) + field(:llm_api_url, :string, default: nil) + field(:llm_course_level_prompt, :string, default: nil) field(:source_chapter, :integer) field(:source_variant, :string) field(:module_help_text, :string) @@ -48,7 +52,7 @@ defmodule Cadet.Courses.Course do @required_fields ~w(course_name viewable enable_game enable_achievements enable_sourcecast enable_stories source_chapter source_variant)a - @optional_fields ~w(course_short_name module_help_text enable_llm_grading llm_api_key)a + @optional_fields ~w(course_short_name module_help_text enable_llm_grading llm_api_key llm_model llm_api_url llm_course_level_prompt)a @spec changeset( {map(), map()} diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index eb9c7dbc8..025324c6a 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -82,6 +82,10 @@ defmodule CadetWeb.AICodeAnalysisController do conn |> put_status(:internal_server_error) |> json(%{"error" => "No LLM API URL configured for this course"}) + is_nil(course.llm_course_level_prompt) or course.llm_course_level_prompt == "" -> + conn + |> put_status(:internal_server_error) + |> json(%{"error" => "No course-level prompt configured for this course"}) true -> case Assessments.get_answers_in_submission(submission_id, question_id) do {:ok, {answers, _assessment}} -> @@ -92,7 +96,7 @@ defmodule CadetWeb.AICodeAnalysisController do |> json(%{"error" => "No answer found for the given submission and question_id"}) _ -> # Get head of answers (should only be one answer for given submission and question) - analyze_code(conn, hd(answers), submission_id, question_id, api_key, course.llm_model, course.llm_api_url) + analyze_code(conn, hd(answers), submission_id, question_id, api_key, course.llm_model, course.llm_api_url, course.llm_course_level_prompt) end {:error, {status, message}} -> @@ -129,26 +133,40 @@ defmodule CadetWeb.AICodeAnalysisController do end) end - defp format_answer(answer) do + defp format_student_answer(answer) do + """ + **Student Answer:** + ``` + #{answer.answer["code"] || "N/A"} + ``` + """ + end + + defp format_system_prompt(course_prompt, answer) do + course_prompt <> "\n\n" <> """ - **Question ID: #{answer.question.id || "N/A"}** + **Additional Instructions for this Question:** + #{answer.question.question["llm_prompt"] || "N/A"} **Question:** + ``` #{answer.question.question["content"] || "N/A"} + ``` **Model Solution:** ``` #{answer.question.question["solution"] || "N/A"} ``` - **Student Answer:** - ``` - #{answer.answer["code"] || "N/A"} - ``` - **Autograding Status:** #{answer.autograding_status || "N/A"} **Autograding Results:** #{format_autograding_results(answer.autograding_results)} **Comments:** #{answer.comments || "None"} + + Your output must include only the comment suggestions, separated exclusively by triple pipes ("|||") with no spaces before or after the pipes, and without any additional formatting, bullet points, or extra text. + + For example: "This is a good answer.|||This is a bad answer.|||This is a great + + The student answer will be given below as part of the USer Prompt. """ end @@ -162,76 +180,23 @@ defmodule CadetWeb.AICodeAnalysisController do defp format_autograding_results(results), do: inspect(results) - defp analyze_code(conn, answer, submission_id, question_id, api_key, llm_model, llm_api_url) do - - raw_prompt = """ - The code below is written in Source, a variant of JavaScript that comes with a rich set of built-in constants and functions. Below is a summary of some key built-in entities available in Source: - - Constants: - - Infinity: The special number value representing infinity. - - NaN: The special number value for "not a number." - - undefined: The special value for an undefined variable. - - math_PI: The constant π (approximately 3.14159). - - math_E: Euler's number (approximately 2.71828). - - Functions: - - __access_export__(exports, lookup_name): Searches for a name in an exports data structure. - - accumulate(f, initial, xs): Reduces a list by applying a binary function from right-to-left. - - append(xs, ys): Appends list ys to the end of list xs. - - char_at(s, i): Returns the character at index i of string s. - - display(v, s): Displays value v (optionally preceded by string s) in the console. - - filter(pred, xs): Returns a new list with elements of xs that satisfy the predicate pred. - - for_each(f, xs): Applies function f to each element of the list xs. - - get_time(): Returns the current time in milliseconds. - - is_list(xs): Checks whether xs is a proper list. - - length(xs): Returns the number of elements in list xs. - - list(...): Constructs a list from the provided values. - - map(f, xs): Applies function f to each element of list xs. - - math_abs(x): Returns the absolute value of x. - - math_ceil(x): Rounds x up to the nearest integer. - - math_floor(x): Rounds x down to the nearest integer. - - pair(x, y): A primitive function that makes a pair whose head (first component) is x and whose tail (second component) is y. - - head(xs): Returns the first element of pair xs. - - tail(xs): Returns the second element of pair xs. - - math_random(): Returns a random number between 0 (inclusive) and 1 (exclusive). - - (For a full list of built-in functions and constants, refer to the Source documentation.) - - Analyze the following submitted "Student Answer" (ONLY) against the given information and provide detailed feedback on correctness, readability, efficiency, and possible improvements. Your evaluation should consider both standard JavaScript features and the additional built-in functions unique to Source. - - Provide between 3 and 5 concise comment suggestions, each under 200 words. - - Your output must include only the comment suggestions, separated exclusively by triple pipes ("|||") with no spaces before or after the pipes, and without any additional formatting, bullet points, or extra text. - - Comments and documentation in the code are not necessary for the code, do not penalise based on that, do not suggest to add comments as well. - - Follow the XP scoring guideline provided below in the question prompt, do not be too harsh! - - For example: "This is a good answer.|||This is a bad answer.|||This is a great answer." - """ + defp analyze_code(conn, answer, submission_id, question_id, api_key, llm_model, llm_api_url, course_prompt) do IO.inspect(answer, label: "Answer being analyzed") formatted_answer = - format_answer(answer) + format_student_answer(answer) |> Jason.encode!() IO.inspect("Formatted answer: #{formatted_answer}", label: "Formatted Answer") - llm_prompt = answer.question.question["llm_prompt"] # "llm_prompt" is a string key, so we can't use the (.) operator directly. optional + system_prompt = format_system_prompt(course_prompt, answer) # Combine prompts if llm_prompt exists - prompt = - if llm_prompt && llm_prompt != "" do - raw_prompt <> "Additional Instructions:\n\n" <> llm_prompt <> "\n\n" <> formatted_answer - else - raw_prompt <> "\n" <> formatted_answer - end - input = %{ model: llm_model, messages: [ - %{role: "system", content: "You are an expert software engineer and educator."}, - %{role: "user", content: prompt} + %{role: "system", content: system_prompt}, + %{role: "user", content: formatted_answer} ], } |> Jason.encode!() @@ -250,7 +215,7 @@ defmodule CadetWeb.AICodeAnalysisController do case Jason.decode(body) do {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> IO.inspect(response, label: "Response from OpenAI API") - save_comment(submission_id, question_id, prompt, formatted_answer, response) + save_comment(submission_id, question_id, system_prompt, formatted_answer, response) comments_list = String.split(response, "|||") filtered_comments = @@ -264,7 +229,7 @@ defmodule CadetWeb.AICodeAnalysisController do save_comment( submission_id, question_id, - prompt, + system_prompt, formatted_answer, nil, "Failed to parse response from OpenAI API" @@ -278,7 +243,7 @@ defmodule CadetWeb.AICodeAnalysisController do save_comment( submission_id, question_id, - prompt, + system_prompt, formatted_answer, nil, "API request failed with status #{status}" @@ -289,7 +254,7 @@ defmodule CadetWeb.AICodeAnalysisController do |> json(%{"error" => "API request failed with status #{status}: #{body}"}) {:error, %HTTPoison.Error{reason: reason}} -> - save_comment(submission_id, question_id, prompt, formatted_answer, nil, reason) + save_comment(submission_id, question_id, system_prompt, formatted_answer, nil, reason) json(conn, %{"error" => "HTTP request error: #{inspect(reason)}"}) end end diff --git a/lib/cadet_web/views/courses_view.ex b/lib/cadet_web/views/courses_view.ex index 819f4aebd..5da78b597 100644 --- a/lib/cadet_web/views/courses_view.ex +++ b/lib/cadet_web/views/courses_view.ex @@ -14,6 +14,9 @@ defmodule CadetWeb.CoursesView do enableStories: :enable_stories, enableLlmGrading: :enable_llm_grading, llmApiKey: :llm_api_key, + llmModel: :llm_model, + llmApiUrl: :llm_api_url, + llmCourseLevelPrompt: :llm_course_level_prompt, sourceChapter: :source_chapter, sourceVariant: :source_variant, moduleHelpText: :module_help_text, diff --git a/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs index c2a0b89f8..c7eafbc68 100644 --- a/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs +++ b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs @@ -3,14 +3,17 @@ defmodule Cadet.Repo.Migrations.AddLlmApiKeyToCourses do def up do alter table(:courses) do + add(:course_level_prompt, :text, null: true) add(:llm_api_key, :text, null: true) - add(:llm_model, :text, null: false, default: "gpt-5-mini") - add(:llm_api_url, :text, null: false, default: "https://api.openai.com/v1/chat/completions") + add(:llm_model, :text, null: true) + add(:llm_api_url, :text, null: true) + add(:llm_course_level_prompt, :text, null: true) end end def down do alter table(:courses) do + remove(:llm_course_level_prompt) remove(:llm_api_key) remove(:llm_model) remove(:llm_api_url) From d14c03e192702288b1808b2052c15d8ce5e133d4 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Tue, 30 Sep 2025 16:52:12 +0800 Subject: [PATCH 33/77] Add encryption_key to env --- config/dev.secrets.exs.example | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/config/dev.secrets.exs.example b/config/dev.secrets.exs.example index 131eb0503..44e31b5af 100644 --- a/config/dev.secrets.exs.example +++ b/config/dev.secrets.exs.example @@ -99,12 +99,8 @@ config :cadet, ] config :openai, - # find it at https://platform.openai.com/account/api-keys - api_key: "the actual api key", - # For source academy deployment, leave this as empty string.Ingeneral could find it at https://platform.openai.com/account/org-settings under "Organization ID". - organization_key: "", - # optional, passed to [HTTPoison.Request](https://hexdocs.pm/httpoison/HTTPoison.Request.html) options - http_options: [recv_timeout: 170_0000] + # TODO: Input your own AES-256 encryption key here for encrypting LLM API keys + encryption_key: "" # config :sentry, # dsn: "https://public_key/sentry.io/somethingsomething" From c345e036f3e096e91d6e0c5e7c2864578f8420a2 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Tue, 7 Oct 2025 12:45:53 +0800 Subject: [PATCH 34/77] Do not hardcode formatting instructions --- lib/cadet_web/controllers/generate_ai_comments.ex | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 025324c6a..951235f57 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -162,11 +162,7 @@ defmodule CadetWeb.AICodeAnalysisController do **Autograding Results:** #{format_autograding_results(answer.autograding_results)} **Comments:** #{answer.comments || "None"} - Your output must include only the comment suggestions, separated exclusively by triple pipes ("|||") with no spaces before or after the pipes, and without any additional formatting, bullet points, or extra text. - - For example: "This is a good answer.|||This is a bad answer.|||This is a great - - The student answer will be given below as part of the USer Prompt. + The student answer will be given below as part of the User Prompt. """ end From b782641e7b47ab7eecae64ddaf18b96155ef7bdf Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Tue, 7 Oct 2025 15:36:36 +0800 Subject: [PATCH 35/77] Add Assessment level prompts to the XML --- lib/cadet/assessments/assessment.ex | 3 ++- lib/cadet/assessments/assessments.ex | 9 +++++++++ lib/cadet/jobs/xml_parser.ex | 1 + .../controllers/generate_ai_comments.ex | 19 ++++++++++++++++--- ...8_add_llm_assessment_prompt_assessment.exs | 9 +++++++++ 5 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 priv/repo/migrations/20251007050808_add_llm_assessment_prompt_assessment.exs diff --git a/lib/cadet/assessments/assessment.ex b/lib/cadet/assessments/assessment.ex index db9cc7514..edfce9bb6 100644 --- a/lib/cadet/assessments/assessment.ex +++ b/lib/cadet/assessments/assessment.ex @@ -36,6 +36,7 @@ defmodule Cadet.Assessments.Assessment do field(:max_team_size, :integer, default: 1) field(:has_token_counter, :boolean, default: false) field(:has_voting_features, :boolean, default: false) + field(:llm_assessment_prompt, :string, default: nil) belongs_to(:config, AssessmentConfig) belongs_to(:course, Course) @@ -46,7 +47,7 @@ defmodule Cadet.Assessments.Assessment do @required_fields ~w(title open_at close_at number course_id config_id max_team_size)a @optional_fields ~w(reading summary_short summary_long - is_published story cover_picture access password has_token_counter has_voting_features)a + is_published story cover_picture access password has_token_counter has_voting_features llm_assessment_prompt)a @optional_file_fields ~w(mission_pdf)a def changeset(assessment, params) do diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 9ed7abb3a..d20871fdc 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -2792,4 +2792,13 @@ defmodule Cadet.Assessments do end) end end + + def get_llm_assessment_prompt(question_id) do + from(q in Question, + where: q.id == ^question_id, + join: a in assoc(q, :assessment), + select: a.llm_assessment_prompt + ) + |> Repo.one() + end end diff --git a/lib/cadet/jobs/xml_parser.ex b/lib/cadet/jobs/xml_parser.ex index abf16c8cd..6c6c087c8 100644 --- a/lib/cadet/jobs/xml_parser.ex +++ b/lib/cadet/jobs/xml_parser.ex @@ -98,6 +98,7 @@ defmodule Cadet.Updater.XMLParser do reading: ~x"//READING/text()" |> transform_by(&process_charlist/1), summary_short: ~x"//WEBSUMMARY/text()" |> transform_by(&process_charlist/1), summary_long: ~x"./TEXT/text()" |> transform_by(&process_charlist/1), + llm_assessment_prompt: ~x"./LLM_ASSESSMENT_PROMPT/text()" |> transform_by(&process_charlist/1), password: ~x"//PASSWORD/text()"so |> transform_by(&process_charlist/1) ) |> Map.put(:is_published, false) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 951235f57..b3164dee7 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -95,8 +95,20 @@ defmodule CadetWeb.AICodeAnalysisController do |> put_status(:not_found) |> json(%{"error" => "No answer found for the given submission and question_id"}) _ -> - # Get head of answers (should only be one answer for given submission and question) - analyze_code(conn, hd(answers), submission_id, question_id, api_key, course.llm_model, course.llm_api_url, course.llm_course_level_prompt) + + # Get head of answers (should only be one answer for given submission and question since we filter to only 1 question) + analyze_code( + conn, + hd(answers), + submission_id, + question_id, + api_key, + course.llm_model, + course.llm_api_url, + course.llm_course_level_prompt, + Assessments.get_llm_assessment_prompt(question_id) + ) + end {:error, {status, message}} -> @@ -176,7 +188,7 @@ defmodule CadetWeb.AICodeAnalysisController do defp format_autograding_results(results), do: inspect(results) - defp analyze_code(conn, answer, submission_id, question_id, api_key, llm_model, llm_api_url, course_prompt) do + defp analyze_code(conn, answer, submission_id, question_id, api_key, llm_model, llm_api_url, course_prompt, assessment_prompt) do IO.inspect(answer, label: "Answer being analyzed") formatted_answer = @@ -184,6 +196,7 @@ defmodule CadetWeb.AICodeAnalysisController do |> Jason.encode!() IO.inspect("Formatted answer: #{formatted_answer}", label: "Formatted Answer") + IO.inspect(assessment_prompt, label: "Assessment Prompt") system_prompt = format_system_prompt(course_prompt, answer) # Combine prompts if llm_prompt exists diff --git a/priv/repo/migrations/20251007050808_add_llm_assessment_prompt_assessment.exs b/priv/repo/migrations/20251007050808_add_llm_assessment_prompt_assessment.exs new file mode 100644 index 000000000..9811ed2a7 --- /dev/null +++ b/priv/repo/migrations/20251007050808_add_llm_assessment_prompt_assessment.exs @@ -0,0 +1,9 @@ +defmodule Cadet.Repo.Migrations.AddLlmAssessmentPromptAssessment do + use Ecto.Migration + + def change do + alter table(:assessments) do + add(:llm_assessment_prompt, :text, default: nil) + end + end +end From c4defb952523ff31856c7963e191c360090ee5d2 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Tue, 7 Oct 2025 17:19:25 +0800 Subject: [PATCH 36/77] Return some additional info for composing of prompts --- lib/cadet_web/admin_views/admin_grading_view.ex | 7 +++++-- lib/cadet_web/controllers/generate_ai_comments.ex | 7 +++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/cadet_web/admin_views/admin_grading_view.ex b/lib/cadet_web/admin_views/admin_grading_view.ex index 87184af95..58c3db6ad 100644 --- a/lib/cadet_web/admin_views/admin_grading_view.ex +++ b/lib/cadet_web/admin_views/admin_grading_view.ex @@ -20,7 +20,8 @@ defmodule CadetWeb.AdminGradingView do coverPicture: assessment.cover_picture, number: assessment.number, story: assessment.story, - reading: assessment.reading + reading: assessment.reading, + llm_assessment_prompt: assessment.llm_assessment_prompt, } end @@ -154,7 +155,9 @@ defmodule CadetWeb.AdminGradingView do team: &extract_team_data(&1.submission.team), question: &build_grading_question/1, solution: &(&1.question.question["solution"] || ""), - grade: &build_grade/1 + grade: &build_grade/1, + autogradingStatus: &(&1.autograding_status || ""), + autogradingResults: &(&1.autograding_results || ""), }) end diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index b3164dee7..1e0169fed 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -154,8 +154,8 @@ defmodule CadetWeb.AICodeAnalysisController do """ end - defp format_system_prompt(course_prompt, answer) do - course_prompt <> "\n\n" <> + defp format_system_prompt(course_prompt, assessment_prompt, answer) do + course_prompt || "" <> "\n\n" <> assessment_prompt || "" <> "\n\n" <> """ **Additional Instructions for this Question:** #{answer.question.question["llm_prompt"] || "N/A"} @@ -172,7 +172,6 @@ defmodule CadetWeb.AICodeAnalysisController do **Autograding Status:** #{answer.autograding_status || "N/A"} **Autograding Results:** #{format_autograding_results(answer.autograding_results)} - **Comments:** #{answer.comments || "None"} The student answer will be given below as part of the User Prompt. """ @@ -198,7 +197,7 @@ defmodule CadetWeb.AICodeAnalysisController do IO.inspect("Formatted answer: #{formatted_answer}", label: "Formatted Answer") IO.inspect(assessment_prompt, label: "Assessment Prompt") - system_prompt = format_system_prompt(course_prompt, answer) + system_prompt = format_system_prompt(course_prompt, assessment_prompt, answer) # Combine prompts if llm_prompt exists input = %{ From 04590f45e214079da464e92f1ab2e01fbee0e243 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Tue, 7 Oct 2025 17:28:42 +0800 Subject: [PATCH 37/77] Remove un-used 'save comments' --- .../controllers/generate_ai_comments.ex | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 1e0169fed..739e47ad5 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -286,26 +286,6 @@ defmodule CadetWeb.AICodeAnalysisController do end end - @doc """ - Saves the chosen comments for a submission and question. - Accepts an array of comments in the request body. - """ - def save_chosen_comments(conn, %{ - "submissionid" => submission_id, - "questionid" => question_id, - "comments" => comments - }) do - case AIComments.update_chosen_comments(submission_id, question_id, comments) do - {:ok, _updated_comment} -> - json(conn, %{"status" => "success"}) - - {:error, changeset} -> - conn - |> put_status(:unprocessable_entity) - |> json(%{"error" => "Failed to save chosen comments"}) - end - end - swagger_path :generate_ai_comments do post("/courses/{courseId}/admin/generate-comments/{submissionId}/{questionId}") From 2920dda8ef4567b378a5aca77a3653990c9379f3 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Tue, 14 Oct 2025 20:42:39 +0800 Subject: [PATCH 38/77] Fix existing assessment tests --- .../admin_views/admin_grading_view.ex | 2 -- .../controllers/generate_ai_comments.ex | 6 ---- .../admin_grading_controller_test.exs | 29 +++++++++++++------ .../assessments_controller_test.exs | 1 + .../assessments/assessment_factory.ex | 3 +- .../factories/assessments/question_factory.ex | 3 +- test/support/seeds.ex | 3 +- 7 files changed, 27 insertions(+), 20 deletions(-) diff --git a/lib/cadet_web/admin_views/admin_grading_view.ex b/lib/cadet_web/admin_views/admin_grading_view.ex index 58c3db6ad..ae5220ff4 100644 --- a/lib/cadet_web/admin_views/admin_grading_view.ex +++ b/lib/cadet_web/admin_views/admin_grading_view.ex @@ -156,8 +156,6 @@ defmodule CadetWeb.AdminGradingView do question: &build_grading_question/1, solution: &(&1.question.question["solution"] || ""), grade: &build_grade/1, - autogradingStatus: &(&1.autograding_status || ""), - autogradingResults: &(&1.autograding_results || ""), }) end diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 739e47ad5..422c4bf95 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -189,13 +189,10 @@ defmodule CadetWeb.AICodeAnalysisController do defp analyze_code(conn, answer, submission_id, question_id, api_key, llm_model, llm_api_url, course_prompt, assessment_prompt) do - IO.inspect(answer, label: "Answer being analyzed") formatted_answer = format_student_answer(answer) |> Jason.encode!() - IO.inspect("Formatted answer: #{formatted_answer}", label: "Formatted Answer") - IO.inspect(assessment_prompt, label: "Assessment Prompt") system_prompt = format_system_prompt(course_prompt, assessment_prompt, answer) # Combine prompts if llm_prompt exists @@ -214,7 +211,6 @@ defmodule CadetWeb.AICodeAnalysisController do {"Content-Type", "application/json"} ] - IO.inspect(input, label: "Input to OpenAI API") case HTTPoison.post(llm_api_url, input, headers, timeout: 60_000, recv_timeout: 60_000 @@ -222,7 +218,6 @@ defmodule CadetWeb.AICodeAnalysisController do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> case Jason.decode(body) do {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> - IO.inspect(response, label: "Response from OpenAI API") save_comment(submission_id, question_id, system_prompt, formatted_answer, response) comments_list = String.split(response, "|||") @@ -247,7 +242,6 @@ defmodule CadetWeb.AICodeAnalysisController do end {:ok, %HTTPoison.Response{status_code: status, body: body}} -> - IO.inspect(body, label: "Error response from OpenAI API") save_comment( submission_id, question_id, diff --git a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs index 87ac65e28..e04c3cddc 100644 --- a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs @@ -256,7 +256,9 @@ defmodule CadetWeb.AdminGradingControllerTest do "coverPicture" => assessment.cover_picture, "number" => assessment.number, "story" => assessment.story, - "reading" => assessment.reading + "reading" => assessment.reading, + "llm_assessment_prompt" => + assessment.llm_assessment_prompt, }, "answers" => answers @@ -312,8 +314,10 @@ defmodule CadetWeb.AdminGradingControllerTest do "content" => &1.question.question.content, "answer" => &1.answer.code, "autogradingStatus" => Atom.to_string(&1.autograding_status), - "autogradingResults" => &1.autograding_results + "autogradingResults" => &1.autograding_results, + "llm_prompt" => &1.question.question.llm_prompt, }, + "ai_comments" => nil, "solution" => &1.question.question.solution, "grade" => %{ "xp" => &1.xp, @@ -361,7 +365,8 @@ defmodule CadetWeb.AdminGradingControllerTest do } end, "autogradingStatus" => Atom.to_string(&1.autograding_status), - "autogradingResults" => &1.autograding_results + "autogradingResults" => &1.autograding_results, + "ai_comments" => nil, }, "solution" => "", "grade" => %{ @@ -425,7 +430,8 @@ defmodule CadetWeb.AdminGradingControllerTest do "id" => &1.submission.student.id }, "team" => %{}, - "solution" => "" + "solution" => "", + "ai_comments" => nil, } end ) @@ -1277,7 +1283,8 @@ defmodule CadetWeb.AdminGradingControllerTest do "coverPicture" => assessment.cover_picture, "number" => assessment.number, "story" => assessment.story, - "reading" => assessment.reading + "reading" => assessment.reading, + "llm_assessment_prompt" => assessment.llm_assessment_prompt, }, "answers" => answers @@ -1333,8 +1340,10 @@ defmodule CadetWeb.AdminGradingControllerTest do "content" => &1.question.question.content, "answer" => &1.answer.code, "autogradingStatus" => Atom.to_string(&1.autograding_status), - "autogradingResults" => &1.autograding_results + "autogradingResults" => &1.autograding_results, + "llm_prompt" => &1.question.question.llm_prompt }, + "ai_comments" => nil, "solution" => &1.question.question.solution, "grade" => %{ "xp" => &1.xp, @@ -1382,7 +1391,8 @@ defmodule CadetWeb.AdminGradingControllerTest do } end, "autogradingStatus" => Atom.to_string(&1.autograding_status), - "autogradingResults" => &1.autograding_results + "autogradingResults" => &1.autograding_results, + "ai_comments" => nil }, "solution" => "", "grade" => %{ @@ -1428,7 +1438,8 @@ defmodule CadetWeb.AdminGradingControllerTest do "answer" => nil, "contestEntries" => [], "scoreLeaderboard" => [], - "popularVoteLeaderboard" => [] + "popularVoteLeaderboard" => [], + "ai_comments" => nil }, "grade" => %{ "xp" => &1.xp, @@ -1696,7 +1707,7 @@ defmodule CadetWeb.AdminGradingControllerTest do course: course, config: assessment_config, is_published: true, - max_team_size: 1 + max_team_size: 1, }) questions = diff --git a/test/cadet_web/controllers/assessments_controller_test.exs b/test/cadet_web/controllers/assessments_controller_test.exs index 708b27eb8..f8923fb39 100644 --- a/test/cadet_web/controllers/assessments_controller_test.exs +++ b/test/cadet_web/controllers/assessments_controller_test.exs @@ -371,6 +371,7 @@ defmodule CadetWeb.AssessmentsControllerTest do "solutionTemplate" => &1.question.template, "prepend" => &1.question.prepend, "postpend" => &1.question.postpend, + "llm_prompt" => &1.question.llm_prompt, "testcases" => Enum.map( &1.question.public, diff --git a/test/factories/assessments/assessment_factory.ex b/test/factories/assessments/assessment_factory.ex index e85267418..5dba2955d 100644 --- a/test/factories/assessments/assessment_factory.ex +++ b/test/factories/assessments/assessment_factory.ex @@ -39,7 +39,8 @@ defmodule Cadet.Assessments.AssessmentFactory do open_at: Timex.now(), close_at: Timex.shift(Timex.now(), days: Enum.random(1..30)), is_published: false, - max_team_size: 1 + max_team_size: 1, + llm_assessment_prompt: nil } end end diff --git a/test/factories/assessments/question_factory.ex b/test/factories/assessments/question_factory.ex index cfd088dd1..ba6bc47cd 100644 --- a/test/factories/assessments/question_factory.ex +++ b/test/factories/assessments/question_factory.ex @@ -51,7 +51,8 @@ defmodule Cadet.Assessments.QuestionFactory do answer: Faker.StarWars.character(), program: Faker.Lorem.Shakespeare.king_richard_iii() } - ] + ], + llm_prompt: nil } end diff --git a/test/support/seeds.ex b/test/support/seeds.ex index e263860e1..1f0b998ac 100644 --- a/test/support/seeds.ex +++ b/test/support/seeds.ex @@ -236,7 +236,8 @@ defmodule Cadet.Test.Seeds do display_order: id, assessment: assessment, max_xp: 1000, - show_solution: assessment.config.type == "path" + show_solution: assessment.config.type == "path", + question: build(:programming_question_content) }) end) From 90882792b532b0f4e986137b04205a72c5b54aa6 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Tue, 14 Oct 2025 22:47:57 +0800 Subject: [PATCH 39/77] Fix generate_ai_comments test cases --- lib/cadet/courses/course.ex | 57 +++-- test.txt | 7 + .../ai_code_analysis_controller_test.exs | 233 +++++------------- test/factories/courses/course_factory.ex | 3 +- 4 files changed, 112 insertions(+), 188 deletions(-) create mode 100644 test.txt diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index a84d58cfd..7b2914dae 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -70,33 +70,48 @@ defmodule Cadet.Courses.Course do |> put_encrypted_llm_api_key() end + def encrypt_llm_api_key(llm_api_key) do + secret = Application.get_env(:openai, :encryption_key) + + if is_binary(secret) and byte_size(secret) >= 16 do + # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256 + key = binary_part(secret, 0, min(32, byte_size(secret))) + # Use AES in GCM mode for encryption + iv = :crypto.strong_rand_bytes(16) + + {ciphertext, tag} = + :crypto.crypto_one_time_aead( + :aes_gcm, + key, + iv, + llm_api_key, + "", + true + ) + + # Store both the IV, ciphertext and tag + encrypted = Base.encode64(iv <> tag <> ciphertext) + else + nil + end + end + def put_encrypted_llm_api_key(changeset) do if llm_api_key = get_change(changeset, :llm_api_key) do if is_binary(llm_api_key) and llm_api_key != "" do - secret = Application.get_env(:openai, :encryption_key) - - if is_binary(secret) and byte_size(secret) >= 16 do - # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256 - key = binary_part(secret, 0, min(32, byte_size(secret))) - # Use AES in GCM mode for encryption - iv = :crypto.strong_rand_bytes(16) - - {ciphertext, tag} = - :crypto.crypto_one_time_aead( - :aes_gcm, - key, - iv, - llm_api_key, - "", - true + encrypted = encrypt_llm_api_key(llm_api_key) + case encrypted do + nil -> + add_error( + changeset, + :llm_api_key, + "encryption key is not configured properly, cannot store LLM API key" ) - # Store both the IV, ciphertext and tag - encrypted = iv <> tag <> ciphertext - put_change(changeset, :llm_api_key, Base.encode64(encrypted)) - else - add_error(changeset, :llm_api_key, "encryption key not configured properly") + encrypted -> + put_change(changeset, :llm_api_key, encrypted) end + else # If empty string or nil is provided, don't encrypt but don't add error changeset diff --git a/test.txt b/test.txt new file mode 100644 index 000000000..26500eecc --- /dev/null +++ b/test.txt @@ -0,0 +1,7 @@ + +22:44:01.241 [info] Migrations already up +Running ExUnit with seed: 808401, max_cases: 44 + +. +Finished in 1.7 seconds (0.00s async, 1.7s sync) +1 test, 0 failures diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs index c17e57174..1e74c4b7a 100644 --- a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -2,192 +2,93 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do use CadetWeb.ConnCase alias Cadet.AIComments alias Cadet.AIComments.AIComment + alias Cadet.Courses.Course + alias Cadet.{Repo} setup do - # Clean up test files before each test - log_file = "log/ai_comments.csv" - File.rm(log_file) - :ok - end + course_with_llm = insert(:course, %{enable_llm_grading: true, llm_api_key: Course.encrypt_llm_api_key("test_key"), llm_model: "gpt-5-mini", llm_api_url: "http://testapi.com", llm_course_level_prompt: "Example Prompt"}) + example_assessment = insert(:assessment, %{course: course_with_llm}) + new_submission = insert(:submission, %{ assessment: example_assessment}) + question = insert(:programming_question, %{assessment: example_assessment}) + answer = insert(:answer, %{submission: new_submission, question: question}) + admin_user = insert(:course_registration, %{role: :admin, course: course_with_llm}) + staff_user = insert(:course_registration, %{role: :staff, course: course_with_llm}) + + {:ok, + %{ + admin_user: admin_user, + staff_user: staff_user, + course_with_llm: course_with_llm, + example_assessment: example_assessment, + new_submission: new_submission, + question: question, + answer: answer + } + } - describe "generate_ai_comments" do - test "successfully logs comments to both database and file", %{conn: conn} do + end + describe "GET /v2/courses/:course_id/admin/generate-comments/:submissionid/:questionid" do + test "success with happy path, admin and staff", %{ + conn: conn, + admin_user: admin_user, + staff_user: staff_user, + course_with_llm: course_with_llm, + example_assessment: example_assessment, + new_submission: new_submission, + question: question, + answer: answer + } do # Test data - submission_id = 123 - question_id = 456 - raw_prompt = "Test prompt" - answers_json = ~s({"test": "data"}) - mock_response = "Comment 1|||Comment 2|||Comment 3" - # Make the API call response = conn - |> post( - Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id) - ) + |> sign_in(staff_user.user) + |> post(build_url_generate_ai_comments(course_with_llm.id, new_submission.id, question.id)) + |> json_response(200) + + response = + conn + |> sign_in(admin_user.user) + |> post(build_url_generate_ai_comments(course_with_llm.id, new_submission.id, question.id)) |> json_response(200) # Verify database entry comments = Repo.all(AIComment) assert length(comments) > 0 latest_comment = List.first(comments) - assert latest_comment.submission_id == submission_id - assert latest_comment.question_id == question_id + assert latest_comment.submission_id == new_submission.id + assert latest_comment.question_id == question.id assert latest_comment.raw_prompt != nil assert latest_comment.answers_json != nil - # Verify CSV file - log_file = "log/ai_comments.csv" - assert File.exists?(log_file) - file_content = File.read!(log_file) - - # Check if CSV contains the required data - assert file_content =~ Integer.to_string(submission_id) - assert file_content =~ Integer.to_string(question_id) end - test "logs error when API call fails", %{conn: conn} do - # Test data with invalid submission_id to trigger error - submission_id = -1 - question_id = 456 - - # Make the API call that should fail - response = - conn - |> post( - Routes.ai_code_analysis_path(conn, :generate_ai_comments, submission_id, question_id) - ) - |> json_response(400) - - # Verify error is logged in database - comments = Repo.all(AIComment) - assert length(comments) > 0 - error_log = List.first(comments) - assert error_log.error != nil - assert error_log.submission_id == submission_id - assert error_log.question_id == question_id - - # Verify error is logged in CSV - log_file = "log/ai_comments.csv" - assert File.exists?(log_file) - file_content = File.read!(log_file) - assert file_content =~ Integer.to_string(submission_id) - assert file_content =~ Integer.to_string(question_id) - assert file_content =~ "error" - end + # test "logs error when API call fails", %{conn: conn} do + # # Test data with invalid submission_id to trigger error + # course_1 = courses.course1 + # submission_id = -1 + # question_id = 456 + + # admin_user = insert(:course_registration, %{role: :admin, course: course_1}) + # # Make the API call that should fail + # response = + # conn + # |> sign_in(admin_user.user) + # |> post(build_url_generate_ai_comments(course_1.id, submission_id, question_id)) + # |> json_response(400) + + # # Verify error is logged in database + # comments = Repo.all(AIComment) + # assert length(comments) > 0 + # error_log = List.first(comments) + # assert error_log.error != nil + # assert error_log.submission_id == submission_id + # assert error_log.question_id == question_id + # end end - describe "save_final_comment" do - test "successfully saves final comment", %{conn: conn} do - # First create a comment entry - submission_id = 123 - question_id = 456 - raw_prompt = "Test prompt" - answers_json = ~s({"test": "data"}) - response = "Comment 1|||Comment 2|||Comment 3" - - {:ok, _comment} = - AIComments.create_ai_comment(%{ - submission_id: submission_id, - question_id: question_id, - raw_prompt: raw_prompt, - answers_json: answers_json, - response: response - }) - - # Now save the final comment - final_comment = "This is the chosen final comment" - - response = - conn - |> post( - Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), - %{ - comment: final_comment - } - ) - |> json_response(200) - - assert response["status"] == "success" - - # Verify the comment was saved - comment = Repo.get_by(AIComment, submission_id: submission_id, question_id: question_id) - assert comment.final_comment == final_comment - end - - test "returns error when no comment exists", %{conn: conn} do - submission_id = 999 - question_id = 888 - final_comment = "This comment should not be saved" - - response = - conn - |> post( - Routes.ai_code_analysis_path(conn, :save_final_comment, submission_id, question_id), - %{ - comment: final_comment - } - ) - |> json_response(422) - - assert response["error"] == "Failed to save final comment" - end + defp build_url_generate_ai_comments(course_id, submission_id, question_id) do + "/v2/courses/#{course_id}/admin/generate-comments/#{submission_id}/#{question_id}" end - describe "save_chosen_comments" do - test "successfully saves chosen comments", %{conn: conn} do - # First create a comment entry - submission_id = 123 - question_id = 456 - raw_prompt = "Test prompt" - answers_json = ~s({"test": "data"}) - response = "Comment 1|||Comment 2|||Comment 3" - - {:ok, _comment} = - AIComments.create_ai_comment(%{ - submission_id: submission_id, - question_id: question_id, - raw_prompt: raw_prompt, - answers_json: answers_json, - response: response - }) - - # Now save the chosen comments - chosen_comments = ["Comment 1", "Comment 2"] - - response = - conn - |> post( - Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), - %{ - comments: chosen_comments - } - ) - |> json_response(200) - - assert response["status"] == "success" - - # Verify the chosen comments were saved - comment = Repo.get_by(AIComment, submission_id: submission_id, question_id: question_id) - assert comment.comment_chosen == chosen_comments - end - - test "returns error when no comment exists", %{conn: conn} do - submission_id = 999 - question_id = 888 - chosen_comments = ["Comment 1", "Comment 2"] - - response = - conn - |> post( - Routes.ai_code_analysis_path(conn, :save_chosen_comments, submission_id, question_id), - %{ - comments: chosen_comments - } - ) - |> json_response(422) - - assert response["error"] == "Failed to save chosen comments" - end - end end diff --git a/test/factories/courses/course_factory.ex b/test/factories/courses/course_factory.ex index 0ca2c3ec6..e9d229803 100644 --- a/test/factories/courses/course_factory.ex +++ b/test/factories/courses/course_factory.ex @@ -18,7 +18,8 @@ defmodule Cadet.Courses.CourseFactory do enable_stories: false, source_chapter: 1, source_variant: "default", - module_help_text: "Help Text" + module_help_text: "Help Text", + enable_llm_grading: false } end end From 714476a0a7ff4fb1adc315c71be8f8f069dca888 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Tue, 14 Oct 2025 23:08:26 +0800 Subject: [PATCH 40/77] Fix bug preventing avengers from generating ai comments --- .../admin_controllers/admin_grading_controller.ex | 11 ++++++++++- lib/cadet_web/admin_views/admin_grading_view.ex | 6 ++++-- lib/cadet_web/controllers/courses_controller.ex | 9 +++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/lib/cadet_web/admin_controllers/admin_grading_controller.ex b/lib/cadet_web/admin_controllers/admin_grading_controller.ex index 9e7507bd7..9c97bc323 100644 --- a/lib/cadet_web/admin_controllers/admin_grading_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_grading_controller.ex @@ -3,6 +3,7 @@ defmodule CadetWeb.AdminGradingController do use PhoenixSwagger alias Cadet.Assessments + alias Cadet.Courses @doc """ # Query Parameters @@ -72,7 +73,15 @@ defmodule CadetWeb.AdminGradingController do def show(conn, %{"submissionid" => submission_id}) when is_ecto_id(submission_id) do case Assessments.get_answers_in_submission(submission_id) do {:ok, {answers, assessment}} -> - render(conn, "show.json", answers: answers, assessment: assessment) + case Courses.get_course_config(assessment.course_id) do + {:ok, course} -> + render(conn, "show.json", course: course, answers: answers, assessment: assessment) + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end {:error, {status, message}} -> conn diff --git a/lib/cadet_web/admin_views/admin_grading_view.ex b/lib/cadet_web/admin_views/admin_grading_view.ex index ae5220ff4..dec67300c 100644 --- a/lib/cadet_web/admin_views/admin_grading_view.ex +++ b/lib/cadet_web/admin_views/admin_grading_view.ex @@ -3,11 +3,12 @@ defmodule CadetWeb.AdminGradingView do import CadetWeb.AssessmentsHelpers - def render("show.json", %{answers: answers, assessment: assessment}) do + def render("show.json", %{course: course, answers: answers, assessment: assessment}) do %{ assessment: render_one(assessment, CadetWeb.AdminGradingView, "assessment.json", as: :assessment), - answers: render_many(answers, CadetWeb.AdminGradingView, "grading_info.json", as: :answer) + answers: render_many(answers, CadetWeb.AdminGradingView, "grading_info.json", as: :answer), + enable_llm_grading: course.enable_llm_grading, } end @@ -22,6 +23,7 @@ defmodule CadetWeb.AdminGradingView do story: assessment.story, reading: assessment.reading, llm_assessment_prompt: assessment.llm_assessment_prompt, + } end diff --git a/lib/cadet_web/controllers/courses_controller.ex b/lib/cadet_web/controllers/courses_controller.ex index c717e3f06..b46226b41 100644 --- a/lib/cadet_web/controllers/courses_controller.ex +++ b/lib/cadet_web/controllers/courses_controller.ex @@ -58,6 +58,9 @@ defmodule CadetWeb.CoursesController do enable_stories(:body, :boolean, "Enable stories", required: true) enable_llm_grading(:body, :boolean, "Enable LLM grading", required: false) llm_api_key(:body, :string, "OpenAI API key for this course", required: false) + llm_model(:body, :string, "LLM model to be used for this course", required: false) + llm_api_url(:body, :string, "LLM API URL to be used for this course", required: false) + llm_course_level_prompt(:body, :string, "LLM course level prompt to be used for this course", required: false) source_chapter(:body, :number, "Default source chapter", required: true) source_variant(:body, Schema.ref(:SourceVariant), "Default source variant name", @@ -101,6 +104,9 @@ defmodule CadetWeb.CoursesController do enable_stories(:boolean, "Enable stories", required: true) enable_llm_grading(:boolean, "Enable LLM grading", required: false) llm_api_key(:string, "OpenAI API key for this course", required: false) + llm_model(:string, "LLM model to be used for this course", required: false) + llm_api_url(:string, "LLM API URL to be used for this course", required: false) + llm_course_level_prompt(:string, "LLM course level prompt to be used for this course", required: false) source_chapter(:integer, "Source Chapter number from 1 to 4", required: true) source_variant(Schema.ref(:SourceVariant), "Source Variant name", required: true) module_help_text(:string, "Module help text", required: true) @@ -117,6 +123,9 @@ defmodule CadetWeb.CoursesController do enable_stories: false, enable_llm_grading: false, llm_api_key: "sk-1234567890", + llm_model: "gpt-4", + llm_api_url: "https://api.openai.com/v1/chat/completions", + llm_course_level_prompt: "You are a helpful teaching assistant for an introductory programming course", source_chapter: 1, source_variant: "default", module_help_text: "Help text", From 5573a21e8f83f260f77baf24e24a9348776fbaeb Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Tue, 14 Oct 2025 23:53:07 +0800 Subject: [PATCH 41/77] Fix up tests + error msgs --- .../controllers/generate_ai_comments.ex | 44 +++++--- test.txt | 7 -- .../ai_code_analysis_controller_test.exs | 104 +++++++++++++----- 3 files changed, 105 insertions(+), 50 deletions(-) delete mode 100644 test.txt diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 422c4bf95..65ca8b35c 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -73,19 +73,19 @@ defmodule CadetWeb.AICodeAnalysisController do is_nil(api_key) -> conn |> put_status(:internal_server_error) - |> json(%{"error" => "No OpenAI API key configured"}) + |> text("No OpenAI API key configured") is_nil(course.llm_model) or course.llm_model == "" -> conn |> put_status(:internal_server_error) - |> json(%{"error" => "No LLM model configured for this course"}) + |> text("No LLM model configured for this course") is_nil(course.llm_api_url) or course.llm_api_url == "" -> conn |> put_status(:internal_server_error) - |> json(%{"error" => "No LLM API URL configured for this course"}) + |> text("No LLM API URL configured for this course") is_nil(course.llm_course_level_prompt) or course.llm_course_level_prompt == "" -> conn |> put_status(:internal_server_error) - |> json(%{"error" => "No course-level prompt configured for this course"}) + |> text("No course-level prompt configured for this course") true -> case Assessments.get_answers_in_submission(submission_id, question_id) do {:ok, {answers, _assessment}} -> @@ -93,7 +93,7 @@ defmodule CadetWeb.AICodeAnalysisController do [] -> conn |> put_status(:not_found) - |> json(%{"error" => "No answer found for the given submission and question_id"}) + |> text("No answer found for the given submission and question_id") _ -> # Get head of answers (should only be one answer for given submission and question since we filter to only 1 question) @@ -121,7 +121,7 @@ defmodule CadetWeb.AICodeAnalysisController do else conn |> put_status(:forbidden) - |> json(%{"error" => "LLM grading is not enabled for this course"}) + |> text("LLM grading is not enabled for this course") end {:error, {status, message}} -> @@ -187,6 +187,13 @@ defmodule CadetWeb.AICodeAnalysisController do defp format_autograding_results(results), do: inspect(results) + def call_llm_endpoint(llm_api_url, input, headers) do + HTTPoison.post(llm_api_url, input, headers, + timeout: 60_000, + recv_timeout: 60_000 + ) + end + defp analyze_code(conn, answer, submission_id, question_id, api_key, llm_model, llm_api_url, course_prompt, assessment_prompt) do formatted_answer = @@ -211,11 +218,8 @@ defmodule CadetWeb.AICodeAnalysisController do {"Content-Type", "application/json"} ] - case HTTPoison.post(llm_api_url, input, headers, - timeout: 60_000, - recv_timeout: 60_000 - ) do - {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> + case call_llm_endpoint(llm_api_url, input, headers) do + {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> case Jason.decode(body) do {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> save_comment(submission_id, question_id, system_prompt, formatted_answer, response) @@ -228,7 +232,7 @@ defmodule CadetWeb.AICodeAnalysisController do json(conn, %{"comments" => filtered_comments}) - {:error, _} -> + {:error, err} -> save_comment( submission_id, question_id, @@ -238,7 +242,9 @@ defmodule CadetWeb.AICodeAnalysisController do "Failed to parse response from OpenAI API" ) - json(conn, %{"error" => "Failed to parse response from OpenAI API"}) + conn + |> put_status(:internal_server_error) + |> text("Failed to parse response from OpenAI API") end {:ok, %HTTPoison.Response{status_code: status, body: body}} -> @@ -253,12 +259,16 @@ defmodule CadetWeb.AICodeAnalysisController do conn |> put_status(:internal_server_error) - |> json(%{"error" => "API request failed with status #{status}: #{body}"}) + |> text("API request failed with status #{status}: #{body}") {:error, %HTTPoison.Error{reason: reason}} -> save_comment(submission_id, question_id, system_prompt, formatted_answer, nil, reason) - json(conn, %{"error" => "HTTP request error: #{inspect(reason)}"}) - end + + conn + |> put_status(:internal_server_error) + |> text( "HTTP request error: #{inspect(reason)}") + end + end @doc """ @@ -276,7 +286,7 @@ defmodule CadetWeb.AICodeAnalysisController do {:error, changeset} -> conn |> put_status(:unprocessable_entity) - |> json(%{"error" => "Failed to save final comment"}) + |> text("Failed to save final comment") end end diff --git a/test.txt b/test.txt deleted file mode 100644 index 26500eecc..000000000 --- a/test.txt +++ /dev/null @@ -1,7 +0,0 @@ - -22:44:01.241 [info] Migrations already up -Running ExUnit with seed: 808401, max_cases: 44 - -. -Finished in 1.7 seconds (0.00s async, 1.7s sync) -1 test, 0 failures diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs index 1e74c4b7a..a6a9857af 100644 --- a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -1,3 +1,4 @@ +import Mock defmodule CadetWeb.AICodeAnalysisControllerTest do use CadetWeb.ConnCase alias Cadet.AIComments @@ -38,20 +39,31 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do question: question, answer: answer } do - # Test data + # Make the API call - response = + with_mock HTTPoison, [:passthrough], + post: fn _url, _body, _headers, _opts -> + {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(%{"choices" => [%{"message" => %{"content" => "Comment1|||Comment2"}}]})}} + end do conn |> sign_in(staff_user.user) |> post(build_url_generate_ai_comments(course_with_llm.id, new_submission.id, question.id)) |> json_response(200) + end - response = - conn + with_mock HTTPoison, [:passthrough], + post: fn _url, _body, _headers, _opts -> + {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(%{"choices" => [%{"message" => %{"content" => "Comment1|||Comment2"}}]})}} + end do + response = conn |> sign_in(admin_user.user) |> post(build_url_generate_ai_comments(course_with_llm.id, new_submission.id, question.id)) |> json_response(200) + # Verify response + assert response["comments"] == ["Comment1", "Comment2"] + end + # Verify database entry comments = Repo.all(AIComment) assert length(comments) > 0 @@ -61,30 +73,70 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do assert latest_comment.raw_prompt != nil assert latest_comment.answers_json != nil + + end - # test "logs error when API call fails", %{conn: conn} do - # # Test data with invalid submission_id to trigger error - # course_1 = courses.course1 - # submission_id = -1 - # question_id = 456 - - # admin_user = insert(:course_registration, %{role: :admin, course: course_1}) - # # Make the API call that should fail - # response = - # conn - # |> sign_in(admin_user.user) - # |> post(build_url_generate_ai_comments(course_1.id, submission_id, question_id)) - # |> json_response(400) - - # # Verify error is logged in database - # comments = Repo.all(AIComment) - # assert length(comments) > 0 - # error_log = List.first(comments) - # assert error_log.error != nil - # assert error_log.submission_id == submission_id - # assert error_log.question_id == question_id - # end + test "errors out when given an invalid submission", %{ + conn: conn, + admin_user: admin_user, + staff_user: staff_user, + course_with_llm: course_with_llm, + example_assessment: example_assessment, + new_submission: new_submission, + question: question, + answer: answer + } do + + random_submission_id = 123 + + # Make the API call that should fail + with_mock HTTPoison, [:passthrough], + post: fn _url, _body, _headers, _opts -> + {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(%{"choices" => [%{"message" => %{"content" => "Comment1|||Comment2"}}]})}} + end do + response = + conn + |> sign_in(admin_user.user) + |> post(build_url_generate_ai_comments(course_with_llm.id, random_submission_id, question.id)) + |> text_response(400) + end + end + + test "LLM endpoint returns an invalid response - should log errors in database", %{ + conn: conn, + admin_user: admin_user, + staff_user: staff_user, + course_with_llm: course_with_llm, + example_assessment: example_assessment, + new_submission: new_submission, + question: question, + answer: answer + } do + + random_submission_id = 123 + + # Make the API call that should fail + with_mock HTTPoison, [:passthrough], + post: fn _url, _body, _headers, _opts -> + {:ok, %HTTPoison.Response{status_code: 200, body: "invalid response"}} + end do + response = + conn + |> sign_in(admin_user.user) + |> post(build_url_generate_ai_comments(course_with_llm.id, new_submission.id, question.id)) + |> text_response(500) + end + + # Verify database entry even with error + comments = Repo.all(AIComment) + assert length(comments) > 0 + latest_comment = List.first(comments) + assert latest_comment.submission_id == new_submission.id + assert latest_comment.question_id == question.id + assert latest_comment.raw_prompt != nil + assert latest_comment.answers_json != nil + end end defp build_url_generate_ai_comments(course_id, submission_id, question_id) do From efc4c570dd4c27db518be8872e6c7bb8b5f33938 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Tue, 14 Oct 2025 23:54:10 +0800 Subject: [PATCH 42/77] Formatting --- lib/cadet/assessments/assessments.ex | 15 +- lib/cadet/courses/course.ex | 2 +- lib/cadet/jobs/xml_parser.ex | 3 +- .../admin_views/admin_grading_view.ex | 13 +- .../controllers/courses_controller.ex | 18 +- .../controllers/generate_ai_comments.ex | 227 +++++++++--------- .../admin_grading_controller_test.exs | 13 +- .../ai_code_analysis_controller_test.exs | 109 ++++++--- 8 files changed, 228 insertions(+), 172 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index d20871fdc..69f252ce3 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -2290,7 +2290,8 @@ defmodule Cadet.Assessments do base_query = Answer |> where(submission_id: ^id) - |> join(:inner, [a], q in assoc(a, :question)) # [a] are bindings (in SQL it is similar to FROM answers "AS a"), this line's alias is INNER JOIN ... "AS q" + # [a] are bindings (in SQL it is similar to FROM answers "AS a"), this line's alias is INNER JOIN ... "AS q" + |> join(:inner, [a], q in assoc(a, :question)) |> join(:inner, [_, q], ast in assoc(q, :assessment)) |> join(:inner, [..., ast], ac in assoc(ast, :config)) |> join(:left, [a, ...], g in assoc(a, :grader)) @@ -2794,11 +2795,11 @@ defmodule Cadet.Assessments do end def get_llm_assessment_prompt(question_id) do - from(q in Question, - where: q.id == ^question_id, - join: a in assoc(q, :assessment), - select: a.llm_assessment_prompt - ) - |> Repo.one() + from(q in Question, + where: q.id == ^question_id, + join: a in assoc(q, :assessment), + select: a.llm_assessment_prompt + ) + |> Repo.one() end end diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index 7b2914dae..4839c2b6b 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -100,6 +100,7 @@ defmodule Cadet.Courses.Course do if llm_api_key = get_change(changeset, :llm_api_key) do if is_binary(llm_api_key) and llm_api_key != "" do encrypted = encrypt_llm_api_key(llm_api_key) + case encrypted do nil -> add_error( @@ -111,7 +112,6 @@ defmodule Cadet.Courses.Course do encrypted -> put_change(changeset, :llm_api_key, encrypted) end - else # If empty string or nil is provided, don't encrypt but don't add error changeset diff --git a/lib/cadet/jobs/xml_parser.ex b/lib/cadet/jobs/xml_parser.ex index 6c6c087c8..d6b87a5c6 100644 --- a/lib/cadet/jobs/xml_parser.ex +++ b/lib/cadet/jobs/xml_parser.ex @@ -98,7 +98,8 @@ defmodule Cadet.Updater.XMLParser do reading: ~x"//READING/text()" |> transform_by(&process_charlist/1), summary_short: ~x"//WEBSUMMARY/text()" |> transform_by(&process_charlist/1), summary_long: ~x"./TEXT/text()" |> transform_by(&process_charlist/1), - llm_assessment_prompt: ~x"./LLM_ASSESSMENT_PROMPT/text()" |> transform_by(&process_charlist/1), + llm_assessment_prompt: + ~x"./LLM_ASSESSMENT_PROMPT/text()" |> transform_by(&process_charlist/1), password: ~x"//PASSWORD/text()"so |> transform_by(&process_charlist/1) ) |> Map.put(:is_published, false) diff --git a/lib/cadet_web/admin_views/admin_grading_view.ex b/lib/cadet_web/admin_views/admin_grading_view.ex index dec67300c..50fcb617f 100644 --- a/lib/cadet_web/admin_views/admin_grading_view.ex +++ b/lib/cadet_web/admin_views/admin_grading_view.ex @@ -8,7 +8,7 @@ defmodule CadetWeb.AdminGradingView do assessment: render_one(assessment, CadetWeb.AdminGradingView, "assessment.json", as: :assessment), answers: render_many(answers, CadetWeb.AdminGradingView, "grading_info.json", as: :answer), - enable_llm_grading: course.enable_llm_grading, + enable_llm_grading: course.enable_llm_grading } end @@ -22,8 +22,7 @@ defmodule CadetWeb.AdminGradingView do number: assessment.number, story: assessment.story, reading: assessment.reading, - llm_assessment_prompt: assessment.llm_assessment_prompt, - + llm_assessment_prompt: assessment.llm_assessment_prompt } end @@ -157,7 +156,7 @@ defmodule CadetWeb.AdminGradingView do team: &extract_team_data(&1.submission.team), question: &build_grading_question/1, solution: &(&1.question.question["solution"] || ""), - grade: &build_grade/1, + grade: &build_grade/1 }) end @@ -166,8 +165,10 @@ defmodule CadetWeb.AdminGradingView do end defp extract_ai_comments_per_answer(question_id, ai_comments) do - matching_comment = ai_comments - |> Enum.find(&(&1.question_id == question_id)) # Equivalent to fn comment -> comment.question_id == question_id end + matching_comment = + ai_comments + # Equivalent to fn comment -> comment.question_id == question_id end + |> Enum.find(&(&1.question_id == question_id)) case matching_comment do nil -> nil diff --git a/lib/cadet_web/controllers/courses_controller.ex b/lib/cadet_web/controllers/courses_controller.ex index b46226b41..c2a18f80e 100644 --- a/lib/cadet_web/controllers/courses_controller.ex +++ b/lib/cadet_web/controllers/courses_controller.ex @@ -60,7 +60,14 @@ defmodule CadetWeb.CoursesController do llm_api_key(:body, :string, "OpenAI API key for this course", required: false) llm_model(:body, :string, "LLM model to be used for this course", required: false) llm_api_url(:body, :string, "LLM API URL to be used for this course", required: false) - llm_course_level_prompt(:body, :string, "LLM course level prompt to be used for this course", required: false) + + llm_course_level_prompt( + :body, + :string, + "LLM course level prompt to be used for this course", + required: false + ) + source_chapter(:body, :number, "Default source chapter", required: true) source_variant(:body, Schema.ref(:SourceVariant), "Default source variant name", @@ -106,7 +113,11 @@ defmodule CadetWeb.CoursesController do llm_api_key(:string, "OpenAI API key for this course", required: false) llm_model(:string, "LLM model to be used for this course", required: false) llm_api_url(:string, "LLM API URL to be used for this course", required: false) - llm_course_level_prompt(:string, "LLM course level prompt to be used for this course", required: false) + + llm_course_level_prompt(:string, "LLM course level prompt to be used for this course", + required: false + ) + source_chapter(:integer, "Source Chapter number from 1 to 4", required: true) source_variant(Schema.ref(:SourceVariant), "Source Variant name", required: true) module_help_text(:string, "Module help text", required: true) @@ -125,7 +136,8 @@ defmodule CadetWeb.CoursesController do llm_api_key: "sk-1234567890", llm_model: "gpt-4", llm_api_url: "https://api.openai.com/v1/chat/completions", - llm_course_level_prompt: "You are a helpful teaching assistant for an introductory programming course", + llm_course_level_prompt: + "You are a helpful teaching assistant for an introductory programming course", source_chapter: 1, source_variant: "default", module_help_text: "Help text", diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 65ca8b35c..c2557d7e2 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -15,7 +15,7 @@ defmodule CadetWeb.AICodeAnalysisController do raw_prompt: raw_prompt, answers_json: answers_json, response: response, - error: error, + error: error } # Check if a comment already exists for the given submission_id and question_id @@ -74,49 +74,51 @@ defmodule CadetWeb.AICodeAnalysisController do conn |> put_status(:internal_server_error) |> text("No OpenAI API key configured") + is_nil(course.llm_model) or course.llm_model == "" -> conn |> put_status(:internal_server_error) |> text("No LLM model configured for this course") + is_nil(course.llm_api_url) or course.llm_api_url == "" -> conn |> put_status(:internal_server_error) |> text("No LLM API URL configured for this course") + is_nil(course.llm_course_level_prompt) or course.llm_course_level_prompt == "" -> conn |> put_status(:internal_server_error) |> text("No course-level prompt configured for this course") + true -> case Assessments.get_answers_in_submission(submission_id, question_id) do - {:ok, {answers, _assessment}} -> - case answers do - [] -> - conn - |> put_status(:not_found) - |> text("No answer found for the given submission and question_id") - _ -> - - # Get head of answers (should only be one answer for given submission and question since we filter to only 1 question) - analyze_code( - conn, - hd(answers), - submission_id, - question_id, - api_key, - course.llm_model, - course.llm_api_url, - course.llm_course_level_prompt, - Assessments.get_llm_assessment_prompt(question_id) + {:ok, {answers, _assessment}} -> + case answers do + [] -> + conn + |> put_status(:not_found) + |> text("No answer found for the given submission and question_id") + + _ -> + # Get head of answers (should only be one answer for given submission and question since we filter to only 1 question) + analyze_code( + conn, + hd(answers), + submission_id, + question_id, + api_key, + course.llm_model, + course.llm_api_url, + course.llm_course_level_prompt, + Assessments.get_llm_assessment_prompt(question_id) ) + end - end - - {:error, {status, message}} -> - conn - |> put_status(status) - |> text(message) - end - + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end end else conn @@ -155,26 +157,28 @@ defmodule CadetWeb.AICodeAnalysisController do end defp format_system_prompt(course_prompt, assessment_prompt, answer) do - course_prompt || "" <> "\n\n" <> assessment_prompt || "" <> "\n\n" <> - """ - **Additional Instructions for this Question:** - #{answer.question.question["llm_prompt"] || "N/A"} - - **Question:** - ``` - #{answer.question.question["content"] || "N/A"} - ``` - - **Model Solution:** - ``` - #{answer.question.question["solution"] || "N/A"} - ``` - - **Autograding Status:** #{answer.autograding_status || "N/A"} - **Autograding Results:** #{format_autograding_results(answer.autograding_results)} - - The student answer will be given below as part of the User Prompt. - """ + course_prompt || "" <> "\n\n" <> assessment_prompt || + "" <> + "\n\n" <> + """ + **Additional Instructions for this Question:** + #{answer.question.question["llm_prompt"] || "N/A"} + + **Question:** + ``` + #{answer.question.question["content"] || "N/A"} + ``` + + **Model Solution:** + ``` + #{answer.question.question["solution"] || "N/A"} + ``` + + **Autograding Status:** #{answer.autograding_status || "N/A"} + **Autograding Results:** #{format_autograding_results(answer.autograding_results)} + + The student answer will be given below as part of the User Prompt. + """ end defp format_autograding_results(nil), do: "N/A" @@ -189,87 +193,94 @@ defmodule CadetWeb.AICodeAnalysisController do def call_llm_endpoint(llm_api_url, input, headers) do HTTPoison.post(llm_api_url, input, headers, - timeout: 60_000, - recv_timeout: 60_000 - ) + timeout: 60_000, + recv_timeout: 60_000 + ) end - defp analyze_code(conn, answer, submission_id, question_id, api_key, llm_model, llm_api_url, course_prompt, assessment_prompt) do - - formatted_answer = - format_student_answer(answer) - |> Jason.encode!() + defp analyze_code( + conn, + answer, + submission_id, + question_id, + api_key, + llm_model, + llm_api_url, + course_prompt, + assessment_prompt + ) do + formatted_answer = + format_student_answer(answer) + |> Jason.encode!() + + system_prompt = format_system_prompt(course_prompt, assessment_prompt, answer) + # Combine prompts if llm_prompt exists + input = + %{ + model: llm_model, + messages: [ + %{role: "system", content: system_prompt}, + %{role: "user", content: formatted_answer} + ] + } + |> Jason.encode!() + headers = [ + {"Authorization", "Bearer #{api_key}"}, + {"Content-Type", "application/json"} + ] - system_prompt = format_system_prompt(course_prompt, assessment_prompt, answer) - # Combine prompts if llm_prompt exists - input = - %{ - model: llm_model, - messages: [ - %{role: "system", content: system_prompt}, - %{role: "user", content: formatted_answer} - ], - } - |> Jason.encode!() + case call_llm_endpoint(llm_api_url, input, headers) do + {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> + case Jason.decode(body) do + {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> + save_comment(submission_id, question_id, system_prompt, formatted_answer, response) + comments_list = String.split(response, "|||") - headers = [ - {"Authorization", "Bearer #{api_key}"}, - {"Content-Type", "application/json"} - ] + filtered_comments = + Enum.filter(comments_list, fn comment -> + String.trim(comment) != "" + end) - case call_llm_endpoint(llm_api_url, input, headers) do - {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> - case Jason.decode(body) do - {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> - save_comment(submission_id, question_id, system_prompt, formatted_answer, response) - comments_list = String.split(response, "|||") - - filtered_comments = - Enum.filter(comments_list, fn comment -> - String.trim(comment) != "" - end) - - json(conn, %{"comments" => filtered_comments}) - - {:error, err} -> - save_comment( - submission_id, - question_id, - system_prompt, - formatted_answer, - nil, - "Failed to parse response from OpenAI API" - ) - - conn - |> put_status(:internal_server_error) - |> text("Failed to parse response from OpenAI API") - end + json(conn, %{"comments" => filtered_comments}) - {:ok, %HTTPoison.Response{status_code: status, body: body}} -> + {:error, err} -> save_comment( submission_id, question_id, system_prompt, formatted_answer, nil, - "API request failed with status #{status}" + "Failed to parse response from OpenAI API" ) conn |> put_status(:internal_server_error) - |> text("API request failed with status #{status}: #{body}") + |> text("Failed to parse response from OpenAI API") + end - {:error, %HTTPoison.Error{reason: reason}} -> - save_comment(submission_id, question_id, system_prompt, formatted_answer, nil, reason) + {:ok, %HTTPoison.Response{status_code: status, body: body}} -> + save_comment( + submission_id, + question_id, + system_prompt, + formatted_answer, + nil, + "API request failed with status #{status}" + ) - conn - |> put_status(:internal_server_error) - |> text( "HTTP request error: #{inspect(reason)}") - end + conn + |> put_status(:internal_server_error) + |> text("API request failed with status #{status}: #{body}") + + {:error, %HTTPoison.Error{reason: reason}} -> + save_comment(submission_id, question_id, system_prompt, formatted_answer, nil, reason) + conn + |> put_status(:internal_server_error) + |> text("HTTP request error: #{inspect(reason)}") end + end @doc """ Saves the final comment chosen for a submission. diff --git a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs index e04c3cddc..e7bd29230 100644 --- a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs @@ -257,8 +257,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "number" => assessment.number, "story" => assessment.story, "reading" => assessment.reading, - "llm_assessment_prompt" => - assessment.llm_assessment_prompt, + "llm_assessment_prompt" => assessment.llm_assessment_prompt }, "answers" => answers @@ -315,7 +314,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "answer" => &1.answer.code, "autogradingStatus" => Atom.to_string(&1.autograding_status), "autogradingResults" => &1.autograding_results, - "llm_prompt" => &1.question.question.llm_prompt, + "llm_prompt" => &1.question.question.llm_prompt }, "ai_comments" => nil, "solution" => &1.question.question.solution, @@ -366,7 +365,7 @@ defmodule CadetWeb.AdminGradingControllerTest do end, "autogradingStatus" => Atom.to_string(&1.autograding_status), "autogradingResults" => &1.autograding_results, - "ai_comments" => nil, + "ai_comments" => nil }, "solution" => "", "grade" => %{ @@ -431,7 +430,7 @@ defmodule CadetWeb.AdminGradingControllerTest do }, "team" => %{}, "solution" => "", - "ai_comments" => nil, + "ai_comments" => nil } end ) @@ -1284,7 +1283,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "number" => assessment.number, "story" => assessment.story, "reading" => assessment.reading, - "llm_assessment_prompt" => assessment.llm_assessment_prompt, + "llm_assessment_prompt" => assessment.llm_assessment_prompt }, "answers" => answers @@ -1707,7 +1706,7 @@ defmodule CadetWeb.AdminGradingControllerTest do course: course, config: assessment_config, is_published: true, - max_team_size: 1, + max_team_size: 1 }) questions = diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs index a6a9857af..2264fdb0f 100644 --- a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -1,4 +1,5 @@ import Mock + defmodule CadetWeb.AICodeAnalysisControllerTest do use CadetWeb.ConnCase alias Cadet.AIComments @@ -7,27 +8,34 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do alias Cadet.{Repo} setup do - course_with_llm = insert(:course, %{enable_llm_grading: true, llm_api_key: Course.encrypt_llm_api_key("test_key"), llm_model: "gpt-5-mini", llm_api_url: "http://testapi.com", llm_course_level_prompt: "Example Prompt"}) + course_with_llm = + insert(:course, %{ + enable_llm_grading: true, + llm_api_key: Course.encrypt_llm_api_key("test_key"), + llm_model: "gpt-5-mini", + llm_api_url: "http://testapi.com", + llm_course_level_prompt: "Example Prompt" + }) + example_assessment = insert(:assessment, %{course: course_with_llm}) - new_submission = insert(:submission, %{ assessment: example_assessment}) + new_submission = insert(:submission, %{assessment: example_assessment}) question = insert(:programming_question, %{assessment: example_assessment}) answer = insert(:answer, %{submission: new_submission, question: question}) admin_user = insert(:course_registration, %{role: :admin, course: course_with_llm}) staff_user = insert(:course_registration, %{role: :staff, course: course_with_llm}) {:ok, - %{ - admin_user: admin_user, - staff_user: staff_user, - course_with_llm: course_with_llm, - example_assessment: example_assessment, - new_submission: new_submission, - question: question, - answer: answer - } - } - + %{ + admin_user: admin_user, + staff_user: staff_user, + course_with_llm: course_with_llm, + example_assessment: example_assessment, + new_submission: new_submission, + question: question, + answer: answer + }} end + describe "GET /v2/courses/:course_id/admin/generate-comments/:submissionid/:questionid" do test "success with happy path, admin and staff", %{ conn: conn, @@ -39,26 +47,44 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do question: question, answer: answer } do - # Make the API call with_mock HTTPoison, [:passthrough], - post: fn _url, _body, _headers, _opts -> - {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(%{"choices" => [%{"message" => %{"content" => "Comment1|||Comment2"}}]})}} - end do + post: fn _url, _body, _headers, _opts -> + {:ok, + %HTTPoison.Response{ + status_code: 200, + body: + Jason.encode!(%{ + "choices" => [%{"message" => %{"content" => "Comment1|||Comment2"}}] + }) + }} + end do conn |> sign_in(staff_user.user) - |> post(build_url_generate_ai_comments(course_with_llm.id, new_submission.id, question.id)) + |> post( + build_url_generate_ai_comments(course_with_llm.id, new_submission.id, question.id) + ) |> json_response(200) end with_mock HTTPoison, [:passthrough], - post: fn _url, _body, _headers, _opts -> - {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(%{"choices" => [%{"message" => %{"content" => "Comment1|||Comment2"}}]})}} - end do - response = conn - |> sign_in(admin_user.user) - |> post(build_url_generate_ai_comments(course_with_llm.id, new_submission.id, question.id)) - |> json_response(200) + post: fn _url, _body, _headers, _opts -> + {:ok, + %HTTPoison.Response{ + status_code: 200, + body: + Jason.encode!(%{ + "choices" => [%{"message" => %{"content" => "Comment1|||Comment2"}}] + }) + }} + end do + response = + conn + |> sign_in(admin_user.user) + |> post( + build_url_generate_ai_comments(course_with_llm.id, new_submission.id, question.id) + ) + |> json_response(200) # Verify response assert response["comments"] == ["Comment1", "Comment2"] @@ -72,9 +98,6 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do assert latest_comment.question_id == question.id assert latest_comment.raw_prompt != nil assert latest_comment.answers_json != nil - - - end test "errors out when given an invalid submission", %{ @@ -87,18 +110,26 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do question: question, answer: answer } do - random_submission_id = 123 # Make the API call that should fail with_mock HTTPoison, [:passthrough], - post: fn _url, _body, _headers, _opts -> - {:ok, %HTTPoison.Response{status_code: 200, body: Jason.encode!(%{"choices" => [%{"message" => %{"content" => "Comment1|||Comment2"}}]})}} - end do + post: fn _url, _body, _headers, _opts -> + {:ok, + %HTTPoison.Response{ + status_code: 200, + body: + Jason.encode!(%{ + "choices" => [%{"message" => %{"content" => "Comment1|||Comment2"}}] + }) + }} + end do response = conn |> sign_in(admin_user.user) - |> post(build_url_generate_ai_comments(course_with_llm.id, random_submission_id, question.id)) + |> post( + build_url_generate_ai_comments(course_with_llm.id, random_submission_id, question.id) + ) |> text_response(400) end end @@ -113,18 +144,19 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do question: question, answer: answer } do - random_submission_id = 123 # Make the API call that should fail with_mock HTTPoison, [:passthrough], - post: fn _url, _body, _headers, _opts -> - {:ok, %HTTPoison.Response{status_code: 200, body: "invalid response"}} - end do + post: fn _url, _body, _headers, _opts -> + {:ok, %HTTPoison.Response{status_code: 200, body: "invalid response"}} + end do response = conn |> sign_in(admin_user.user) - |> post(build_url_generate_ai_comments(course_with_llm.id, new_submission.id, question.id)) + |> post( + build_url_generate_ai_comments(course_with_llm.id, new_submission.id, question.id) + ) |> text_response(500) end @@ -142,5 +174,4 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do defp build_url_generate_ai_comments(course_id, submission_id, question_id) do "/v2/courses/#{course_id}/admin/generate-comments/#{submission_id}/#{question_id}" end - end From aa84560bdafc5f0b4c92c71dcdeae3adc86056e9 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Wed, 15 Oct 2025 00:37:06 +0800 Subject: [PATCH 43/77] some mix credo suggestions --- lib/cadet/assessments/assessments.ex | 8 +++++--- lib/cadet_web/controllers/generate_ai_comments.ex | 6 ++++-- .../controllers/ai_code_analysis_controller_test.exs | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index f403419c3..f5c9287a5 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -3050,7 +3050,8 @@ defmodule Cadet.Assessments do base_query = Answer |> where(submission_id: ^id) - # [a] are bindings (in SQL it is similar to FROM answers "AS a"), this line's alias is INNER JOIN ... "AS q" + # [a] are bindings (in SQL it is similar to FROM answers "AS a"), + # this line's alias is INNER JOIN ... "AS q" |> join(:inner, [a], q in assoc(a, :question)) |> join(:inner, [_, q], ast in assoc(q, :assessment)) |> join(:inner, [..., ast], ac in assoc(ast, :config)) @@ -3555,11 +3556,12 @@ defmodule Cadet.Assessments do end def get_llm_assessment_prompt(question_id) do - from(q in Question, + query = from(q in Question, where: q.id == ^question_id, join: a in assoc(q, :assessment), select: a.llm_assessment_prompt ) - |> Repo.one() + + Repo.one(query) end end diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index c2557d7e2..30bba328c 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -100,7 +100,8 @@ defmodule CadetWeb.AICodeAnalysisController do |> text("No answer found for the given submission and question_id") _ -> - # Get head of answers (should only be one answer for given submission and question since we filter to only 1 question) + # Get head of answers (should only be one answer for given submission + # and question since we filter to only 1 question) analyze_code( conn, hd(answers), @@ -210,7 +211,8 @@ defmodule CadetWeb.AICodeAnalysisController do assessment_prompt ) do formatted_answer = - format_student_answer(answer) + answer + |> format_student_answer() |> Jason.encode!() system_prompt = format_system_prompt(course_prompt, assessment_prompt, answer) diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs index 2264fdb0f..f26a661f2 100644 --- a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -5,7 +5,7 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do alias Cadet.AIComments alias Cadet.AIComments.AIComment alias Cadet.Courses.Course - alias Cadet.{Repo} + alias Cadet.Repo setup do course_with_llm = From 4537270da598bd980832d36dcd2136932811ab69 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Wed, 15 Oct 2025 01:11:54 +0800 Subject: [PATCH 44/77] format --- lib/cadet/assessments/assessments.ex | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index f5c9287a5..3e4dc1f99 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -3556,11 +3556,12 @@ defmodule Cadet.Assessments do end def get_llm_assessment_prompt(question_id) do - query = from(q in Question, - where: q.id == ^question_id, - join: a in assoc(q, :assessment), - select: a.llm_assessment_prompt - ) + query = + from(q in Question, + where: q.id == ^question_id, + join: a in assoc(q, :assessment), + select: a.llm_assessment_prompt + ) Repo.one(query) end From 62b6437f3ea5cdfcbe99080f84dc23666269844b Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Wed, 15 Oct 2025 01:24:31 +0800 Subject: [PATCH 45/77] Fix credo issue --- .../controllers/generate_ai_comments.ex | 83 +++++++++++-------- 1 file changed, 48 insertions(+), 35 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 30bba328c..f2d8a6520 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -50,6 +50,25 @@ defmodule CadetWeb.AICodeAnalysisController do end end + defp check_llm_grading_parameters(llm_api_key, llm_model, llm_api_url, llm_course_level_prompt) do + cond do + is_nil(llm_api_key) -> + {:error, "LLM API key is not configured for this course or in the environment"} + + is_nil(llm_model) or llm_model == "" -> + {:error, "LLM model is not configured for this course"} + + is_nil(llm_api_url) or llm_api_url == "" -> + {:error, "LLM API URL is not configured for this course"} + + is_nil(llm_course_level_prompt) or llm_course_level_prompt == "" -> + {:error, "LLM course-level prompt is not configured for this course"} + + true -> + :ok + end + end + @doc """ Fetches the question details and answers based on submissionid and questionid and generates AI-generated comments. """ @@ -69,26 +88,16 @@ defmodule CadetWeb.AICodeAnalysisController do decrypted_api_key = decrypt_llm_api_key(course.llm_api_key) api_key = decrypted_api_key || Application.get_env(:openai, :api_key) - cond do - is_nil(api_key) -> - conn - |> put_status(:internal_server_error) - |> text("No OpenAI API key configured") - - is_nil(course.llm_model) or course.llm_model == "" -> - conn - |> put_status(:internal_server_error) - |> text("No LLM model configured for this course") - - is_nil(course.llm_api_url) or course.llm_api_url == "" -> - conn - |> put_status(:internal_server_error) - |> text("No LLM API URL configured for this course") - - is_nil(course.llm_course_level_prompt) or course.llm_course_level_prompt == "" -> + case check_llm_grading_parameters( + api_key, + course.llm_model, + course.llm_api_url, + course.llm_course_level_prompt + ) do + {:error, error_msg} -> conn - |> put_status(:internal_server_error) - |> text("No course-level prompt configured for this course") + |> put_status(:bad_request) + |> text(error_msg) true -> case Assessments.get_answers_in_submission(submission_id, question_id) do @@ -104,14 +113,16 @@ defmodule CadetWeb.AICodeAnalysisController do # and question since we filter to only 1 question) analyze_code( conn, - hd(answers), - submission_id, - question_id, - api_key, - course.llm_model, - course.llm_api_url, - course.llm_course_level_prompt, - Assessments.get_llm_assessment_prompt(question_id) + %{ + answers: hd(answers), + submission_id: submission_id, + question_id: question_id, + api_key: api_key, + llm_model: course.llm_model, + llm_api_url: course.llm_api_url, + llm_course_level_prompt: course.llm_course_level_prompt, + assessment_prompt: Assessments.get_llm_assessment_prompt(question_id) + } ) end @@ -201,14 +212,16 @@ defmodule CadetWeb.AICodeAnalysisController do defp analyze_code( conn, - answer, - submission_id, - question_id, - api_key, - llm_model, - llm_api_url, - course_prompt, - assessment_prompt + %{ + answer: answer, + submission_id: submission_id, + question_id: question_id, + api_key: api_key, + llm_model: llm_model, + llm_api_url: llm_api_url, + course_prompt: course_prompt, + assessment_prompt: assessment_prompt + } ) do formatted_answer = answer From 6e769fd47d877a71a97b79ba84e01a243fca6e2d Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Wed, 15 Oct 2025 01:34:35 +0800 Subject: [PATCH 46/77] bug fix + credo fixes --- .../admin_controllers/admin_grading_controller.ex | 3 +-- lib/cadet_web/controllers/generate_ai_comments.ex | 8 ++++---- .../controllers/ai_code_analysis_controller_test.exs | 6 ++---- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/cadet_web/admin_controllers/admin_grading_controller.ex b/lib/cadet_web/admin_controllers/admin_grading_controller.ex index 9c97bc323..564988b9f 100644 --- a/lib/cadet_web/admin_controllers/admin_grading_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_grading_controller.ex @@ -2,8 +2,7 @@ defmodule CadetWeb.AdminGradingController do use CadetWeb, :controller use PhoenixSwagger - alias Cadet.Assessments - alias Cadet.Courses + alias Cadet.{Assessments, Courses} @doc """ # Query Parameters diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index f2d8a6520..98b0ed0c0 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -65,7 +65,7 @@ defmodule CadetWeb.AICodeAnalysisController do {:error, "LLM course-level prompt is not configured for this course"} true -> - :ok + {:ok} end end @@ -99,7 +99,7 @@ defmodule CadetWeb.AICodeAnalysisController do |> put_status(:bad_request) |> text(error_msg) - true -> + {:ok} -> case Assessments.get_answers_in_submission(submission_id, question_id) do {:ok, {answers, _assessment}} -> case answers do @@ -114,13 +114,13 @@ defmodule CadetWeb.AICodeAnalysisController do analyze_code( conn, %{ - answers: hd(answers), + answer: hd(answers), submission_id: submission_id, question_id: question_id, api_key: api_key, llm_model: course.llm_model, llm_api_url: course.llm_api_url, - llm_course_level_prompt: course.llm_course_level_prompt, + course_prompt: course.llm_course_level_prompt, assessment_prompt: Assessments.get_llm_assessment_prompt(question_id) } ) diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs index f26a661f2..d80ee2bea 100644 --- a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -2,10 +2,8 @@ import Mock defmodule CadetWeb.AICodeAnalysisControllerTest do use CadetWeb.ConnCase - alias Cadet.AIComments - alias Cadet.AIComments.AIComment - alias Cadet.Courses.Course - alias Cadet.Repo + alias Cadet.{Repo, AIComments} + alias Cadet.{AIComments.AIComment, Courses.Course} setup do course_with_llm = From 1bd9398c1540fab71dc4da4c4f50c938d39fb927 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Wed, 15 Oct 2025 01:57:24 +0800 Subject: [PATCH 47/77] Fix tests --- .../admin_grading_controller_test.exs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs index e7bd29230..03c6e8079 100644 --- a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs @@ -248,6 +248,7 @@ defmodule CadetWeb.AdminGradingControllerTest do conn = get(conn, build_url(course.id, submission.id)) expected = %{ + "enable_llm_grading" => false, "assessment" => %{ "id" => assessment.id, "title" => assessment.title, @@ -365,8 +366,8 @@ defmodule CadetWeb.AdminGradingControllerTest do end, "autogradingStatus" => Atom.to_string(&1.autograding_status), "autogradingResults" => &1.autograding_results, - "ai_comments" => nil }, + "ai_comments" => nil, "solution" => "", "grade" => %{ "xp" => &1.xp, @@ -413,6 +414,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "scoreLeaderboard" => [], "popularVoteLeaderboard" => [] }, + "ai_comments" => nil, "grade" => %{ "xp" => &1.xp, "xpAdjustment" => &1.xp_adjustment, @@ -430,7 +432,6 @@ defmodule CadetWeb.AdminGradingControllerTest do }, "team" => %{}, "solution" => "", - "ai_comments" => nil } end ) @@ -1274,6 +1275,7 @@ defmodule CadetWeb.AdminGradingControllerTest do conn = get(conn, build_url(course.id, submission.id)) expected = %{ + "enable_llm_grading" => false, "assessment" => %{ "id" => assessment.id, "title" => assessment.title, @@ -1391,8 +1393,8 @@ defmodule CadetWeb.AdminGradingControllerTest do end, "autogradingStatus" => Atom.to_string(&1.autograding_status), "autogradingResults" => &1.autograding_results, - "ai_comments" => nil }, + "ai_comments" => nil, "solution" => "", "grade" => %{ "xp" => &1.xp, @@ -1438,8 +1440,8 @@ defmodule CadetWeb.AdminGradingControllerTest do "contestEntries" => [], "scoreLeaderboard" => [], "popularVoteLeaderboard" => [], - "ai_comments" => nil }, + "ai_comments" => nil, "grade" => %{ "xp" => &1.xp, "xpAdjustment" => &1.xp_adjustment, From 4c341d11baa538fea9979a88fda94c16bcc617e1 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Wed, 15 Oct 2025 01:57:35 +0800 Subject: [PATCH 48/77] format --- .../admin_controllers/admin_grading_controller_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs index 03c6e8079..1d766f6a2 100644 --- a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs @@ -365,7 +365,7 @@ defmodule CadetWeb.AdminGradingControllerTest do } end, "autogradingStatus" => Atom.to_string(&1.autograding_status), - "autogradingResults" => &1.autograding_results, + "autogradingResults" => &1.autograding_results }, "ai_comments" => nil, "solution" => "", @@ -431,7 +431,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "id" => &1.submission.student.id }, "team" => %{}, - "solution" => "", + "solution" => "" } end ) @@ -1392,7 +1392,7 @@ defmodule CadetWeb.AdminGradingControllerTest do } end, "autogradingStatus" => Atom.to_string(&1.autograding_status), - "autogradingResults" => &1.autograding_results, + "autogradingResults" => &1.autograding_results }, "ai_comments" => nil, "solution" => "", @@ -1439,7 +1439,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "answer" => nil, "contestEntries" => [], "scoreLeaderboard" => [], - "popularVoteLeaderboard" => [], + "popularVoteLeaderboard" => [] }, "ai_comments" => nil, "grade" => %{ From 2947f3c750f5033130a836514f967bd3753cd03c Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Sun, 26 Oct 2025 22:07:10 +0800 Subject: [PATCH 49/77] Modify test.exs --- config/test.exs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/config/test.exs b/config/test.exs index 30a8f8cc0..346858849 100644 --- a/config/test.exs +++ b/config/test.exs @@ -100,3 +100,7 @@ config :cadet, Oban, testing: :manual config :cadet, Cadet.Mailer, adapter: Bamboo.TestAdapter + +config :openai, + # TODO: Input your own AES-256 encryption key here for encrypting LLM API keys + encryption_key: "b4u7g0AyN3Tu2br9WSdZQjLMQ8bed/wgQWrH2x3qPdW8D55iv10+ySgs+bxDirWE" From 8669e3ad18bb733a9d21a0292be3d737fffabb5e Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Sun, 26 Oct 2025 22:16:36 +0800 Subject: [PATCH 50/77] Update lib/cadet_web/controllers/generate_ai_comments.ex Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- lib/cadet_web/controllers/generate_ai_comments.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 98b0ed0c0..29c1bdc27 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -169,9 +169,8 @@ defmodule CadetWeb.AICodeAnalysisController do end defp format_system_prompt(course_prompt, assessment_prompt, answer) do - course_prompt || "" <> "\n\n" <> assessment_prompt || - "" <> - "\n\n" <> + (course_prompt || "") <> "\n\n" <> (assessment_prompt || "") <> + "\n\n" <> """ **Additional Instructions for this Question:** #{answer.question.question["llm_prompt"] || "N/A"} From 58345d4fd1d1212def1a519edc1857553e9c4092 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Sun, 26 Oct 2025 22:19:18 +0800 Subject: [PATCH 51/77] Copilot feedback --- lib/cadet/courses/course.ex | 3 +-- lib/cadet_web/admin_views/admin_grading_view.ex | 2 +- .../migrations/20240320000001_add_llm_api_key_to_courses.exs | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index dda070172..61810e82a 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -117,8 +117,7 @@ defmodule Cadet.Courses.Course do changeset end else - # The key is not being changed, so we need to preserve the existing value - put_change(changeset, :llm_api_key, changeset.data.llm_api_key) + changeset end end diff --git a/lib/cadet_web/admin_views/admin_grading_view.ex b/lib/cadet_web/admin_views/admin_grading_view.ex index 50fcb617f..3521a1e01 100644 --- a/lib/cadet_web/admin_views/admin_grading_view.ex +++ b/lib/cadet_web/admin_views/admin_grading_view.ex @@ -172,7 +172,7 @@ defmodule CadetWeb.AdminGradingView do case matching_comment do nil -> nil - comment -> %{response: matching_comment.response, insertedAt: matching_comment.inserted_at} + comment -> %{response: comment.response, insertedAt: comment.inserted_at} end end diff --git a/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs index c7eafbc68..6b31894df 100644 --- a/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs +++ b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs @@ -13,6 +13,7 @@ defmodule Cadet.Repo.Migrations.AddLlmApiKeyToCourses do def down do alter table(:courses) do + remove(:course_level_prompt) remove(:llm_course_level_prompt) remove(:llm_api_key) remove(:llm_model) From f794a14f357f7491589d0adbf0d8f21af9422834 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Sun, 26 Oct 2025 22:21:36 +0800 Subject: [PATCH 52/77] format --- .../controllers/generate_ai_comments.ex | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 29c1bdc27..bb104c339 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -169,27 +169,29 @@ defmodule CadetWeb.AICodeAnalysisController do end defp format_system_prompt(course_prompt, assessment_prompt, answer) do - (course_prompt || "") <> "\n\n" <> (assessment_prompt || "") <> + (course_prompt || "") <> "\n\n" <> - """ - **Additional Instructions for this Question:** - #{answer.question.question["llm_prompt"] || "N/A"} + (assessment_prompt || "") <> + "\n\n" <> + """ + **Additional Instructions for this Question:** + #{answer.question.question["llm_prompt"] || "N/A"} - **Question:** - ``` - #{answer.question.question["content"] || "N/A"} - ``` + **Question:** + ``` + #{answer.question.question["content"] || "N/A"} + ``` - **Model Solution:** - ``` - #{answer.question.question["solution"] || "N/A"} - ``` + **Model Solution:** + ``` + #{answer.question.question["solution"] || "N/A"} + ``` - **Autograding Status:** #{answer.autograding_status || "N/A"} - **Autograding Results:** #{format_autograding_results(answer.autograding_results)} + **Autograding Status:** #{answer.autograding_status || "N/A"} + **Autograding Results:** #{format_autograding_results(answer.autograding_results)} - The student answer will be given below as part of the User Prompt. - """ + The student answer will be given below as part of the User Prompt. + """ end defp format_autograding_results(nil), do: "N/A" From a9e2b2a72b073bc3eb86a0a6c13ade5ea8ad2bce Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Mon, 27 Oct 2025 00:57:08 +0800 Subject: [PATCH 53/77] Work on sentry comments --- config/dev.secrets.exs.example | 2 +- lib/cadet/courses/course.ex | 4 +- .../controllers/generate_ai_comments.ex | 153 ++++++++++-------- ...40320000001_add_llm_api_key_to_courses.exs | 2 - .../ai_code_analysis_controller_test.exs | 2 - 5 files changed, 88 insertions(+), 75 deletions(-) diff --git a/config/dev.secrets.exs.example b/config/dev.secrets.exs.example index 44e31b5af..9f42df8fc 100644 --- a/config/dev.secrets.exs.example +++ b/config/dev.secrets.exs.example @@ -99,7 +99,7 @@ config :cadet, ] config :openai, - # TODO: Input your own AES-256 encryption key here for encrypting LLM API keys + # TODO: Input your own AES-256 encryption key here for encrypting LLM API keys of 16, 24 or 32 bytes encryption_key: "" # config :sentry, diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index 61810e82a..a14e87a73 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -92,7 +92,7 @@ defmodule Cadet.Courses.Course do # Store both the IV, ciphertext and tag encrypted = Base.encode64(iv <> tag <> ciphertext) else - nil + {:error, :invalid_encryption_key} end end @@ -102,7 +102,7 @@ defmodule Cadet.Courses.Course do encrypted = encrypt_llm_api_key(llm_api_key) case encrypted do - nil -> + {:error, :invalid_encryption_key} -> add_error( changeset, :llm_api_key, diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index bb104c339..02b983690 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -53,22 +53,30 @@ defmodule CadetWeb.AICodeAnalysisController do defp check_llm_grading_parameters(llm_api_key, llm_model, llm_api_url, llm_course_level_prompt) do cond do is_nil(llm_api_key) -> - {:error, "LLM API key is not configured for this course or in the environment"} + {:parameter_error, "LLM API key is not configured for this course or in the environment"} is_nil(llm_model) or llm_model == "" -> - {:error, "LLM model is not configured for this course"} + {:parameter_error, "LLM model is not configured for this course"} is_nil(llm_api_url) or llm_api_url == "" -> - {:error, "LLM API URL is not configured for this course"} + {:parameter_error, "LLM API URL is not configured for this course"} is_nil(llm_course_level_prompt) or llm_course_level_prompt == "" -> - {:error, "LLM course-level prompt is not configured for this course"} + {:parameter_error, "LLM course-level prompt is not configured for this course"} true -> {:ok} end end + defp ensure_llm_enabled(course) do + if course.enable_llm_grading do + {:ok} + else + {:error, {:forbidden, "LLM grading is not enabled for this course"}} + end + end + @doc """ Fetches the question details and answers based on submissionid and questionid and generates AI-generated comments. """ @@ -78,65 +86,57 @@ defmodule CadetWeb.AICodeAnalysisController do "course_id" => course_id }) when is_ecto_id(submission_id) do - # Check if LLM grading is enabled for this course - question_id = String.to_integer(question_id) - - case Courses.get_course_config(course_id) do - {:ok, course} -> - if course.enable_llm_grading do - # Get API key from course config or fall back to environment variable - decrypted_api_key = decrypt_llm_api_key(course.llm_api_key) - api_key = decrypted_api_key || Application.get_env(:openai, :api_key) - - case check_llm_grading_parameters( - api_key, - course.llm_model, - course.llm_api_url, - course.llm_course_level_prompt - ) do - {:error, error_msg} -> - conn - |> put_status(:bad_request) - |> text(error_msg) - - {:ok} -> - case Assessments.get_answers_in_submission(submission_id, question_id) do - {:ok, {answers, _assessment}} -> - case answers do - [] -> - conn - |> put_status(:not_found) - |> text("No answer found for the given submission and question_id") - - _ -> - # Get head of answers (should only be one answer for given submission - # and question since we filter to only 1 question) - analyze_code( - conn, - %{ - answer: hd(answers), - submission_id: submission_id, - question_id: question_id, - api_key: api_key, - llm_model: course.llm_model, - llm_api_url: course.llm_api_url, - course_prompt: course.llm_course_level_prompt, - assessment_prompt: Assessments.get_llm_assessment_prompt(question_id) - } - ) - end - - {:error, {status, message}} -> - conn - |> put_status(status) - |> text(message) - end - end - else + with {qid, ""} <- Integer.parse(question_id), + {:ok, course} <- Courses.get_course_config(course_id), + {:ok} <- ensure_llm_enabled(course), + {:ok, key} <- decrypt_llm_api_key(course.llm_api_key), + {:ok} <- + check_llm_grading_parameters( + key, + course.llm_model, + course.llm_api_url, + course.llm_course_level_prompt + ), + {:ok, {answers, _}} <- Assessments.get_answers_in_submission(submission_id, qid) do + # Get head of answers (should only be one answer for given submission + # and question since we filter to only 1 question) + case answers do + [] -> conn - |> put_status(:forbidden) - |> text("LLM grading is not enabled for this course") - end + |> put_status(:not_found) + |> text("No answer found for the given submission and question_id") + + _ -> + analyze_code( + conn, + %{ + answer: hd(answers), + submission_id: submission_id, + question_id: qid, + api_key: key, + llm_model: course.llm_model, + llm_api_url: course.llm_api_url, + course_prompt: course.llm_course_level_prompt, + assessment_prompt: Assessments.get_llm_assessment_prompt(qid) + } + ) + end + else + :error -> + conn + |> put_status(:bad_request) + |> text("Invalid question ID format") + + {:decrypt_error, err} -> + conn + |> put_status(:internal_server_error) + |> text("Failed to decrypt LLM API key: #{inspect(err)}") + + # Errors for check_llm_grading_parameters + {:parameter_error, error_msg} -> + conn + |> put_status(:bad_request) + |> text(error_msg) {:error, {status, message}} -> conn @@ -260,6 +260,20 @@ defmodule CadetWeb.AICodeAnalysisController do json(conn, %{"comments" => filtered_comments}) + {:ok, other} -> + save_comment( + submission_id, + question_id, + system_prompt, + formatted_answer, + Jason.encode!(other), + "Unexpected JSON shape" + ) + + conn + |> put_status(:bad_gateway) + |> text("Unexpected response format from OpenAI API") + {:error, err} -> save_comment( submission_id, @@ -393,18 +407,21 @@ defmodule CadetWeb.AICodeAnalysisController do ciphertext = binary_part(decoded, 32, byte_size(decoded) - 32) case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do - plain_text when is_binary(plain_text) -> plain_text - _ -> nil + plain_text when is_binary(plain_text) -> {:ok, plain_text} + _ -> {:decrypt_error, :decryption_failed} end _ -> - Logger.error("Failed to decode encrypted key") - nil + Logger.error( + "Failed to decode encrypted key, is it a valid AES-256 key of 16, 24 or 32 bytes?" + ) + + {:decrypt_error, :decryption_failed} end _ -> - Logger.error("Encryption key not configured properly") - nil + Logger.error("Encryption key not configured") + {:decrypt_error, :invalid_encryption_key} end end end diff --git a/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs index 6b31894df..2c6724822 100644 --- a/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs +++ b/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs @@ -3,7 +3,6 @@ defmodule Cadet.Repo.Migrations.AddLlmApiKeyToCourses do def up do alter table(:courses) do - add(:course_level_prompt, :text, null: true) add(:llm_api_key, :text, null: true) add(:llm_model, :text, null: true) add(:llm_api_url, :text, null: true) @@ -13,7 +12,6 @@ defmodule Cadet.Repo.Migrations.AddLlmApiKeyToCourses do def down do alter table(:courses) do - remove(:course_level_prompt) remove(:llm_course_level_prompt) remove(:llm_api_key) remove(:llm_model) diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs index d80ee2bea..496d949d9 100644 --- a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -142,8 +142,6 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do question: question, answer: answer } do - random_submission_id = 123 - # Make the API call that should fail with_mock HTTPoison, [:passthrough], post: fn _url, _body, _headers, _opts -> From 3e96d6b56824ddd753b2789fe5724ad58ae71786 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Mon, 27 Oct 2025 01:04:53 +0800 Subject: [PATCH 54/77] Fix type --- lib/cadet_web/controllers/generate_ai_comments.ex | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 02b983690..764eeb7f9 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -52,9 +52,6 @@ defmodule CadetWeb.AICodeAnalysisController do defp check_llm_grading_parameters(llm_api_key, llm_model, llm_api_url, llm_course_level_prompt) do cond do - is_nil(llm_api_key) -> - {:parameter_error, "LLM API key is not configured for this course or in the environment"} - is_nil(llm_model) or llm_model == "" -> {:parameter_error, "LLM model is not configured for this course"} From 5bf1aa66d584975563f60b365b53ec495243fef5 Mon Sep 17 00:00:00 2001 From: Richard Dominick <34370238+RichDom2185@users.noreply.github.com> Date: Tue, 28 Oct 2025 16:03:45 +0800 Subject: [PATCH 55/77] Redate migrations to maintain total order --- ...ading_access.exs => 20251008160219_add_llm_grading_access.exs} | 0 ..._courses.exs => 20251019160255_add_llm_api_key_to_courses.exs} | 0 ...eate_ai_comments.exs => 20251022103623_create_ai_comments.exs} | 0 ...xs => 20251028050808_add_llm_assessment_prompt_assessment.exs} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename priv/repo/migrations/{20240320000000_add_llm_grading_access.exs => 20251008160219_add_llm_grading_access.exs} (100%) rename priv/repo/migrations/{20240320000001_add_llm_api_key_to_courses.exs => 20251019160255_add_llm_api_key_to_courses.exs} (100%) rename priv/repo/migrations/{20250220103623_create_ai_comments.exs => 20251022103623_create_ai_comments.exs} (100%) rename priv/repo/migrations/{20251007050808_add_llm_assessment_prompt_assessment.exs => 20251028050808_add_llm_assessment_prompt_assessment.exs} (100%) diff --git a/priv/repo/migrations/20240320000000_add_llm_grading_access.exs b/priv/repo/migrations/20251008160219_add_llm_grading_access.exs similarity index 100% rename from priv/repo/migrations/20240320000000_add_llm_grading_access.exs rename to priv/repo/migrations/20251008160219_add_llm_grading_access.exs diff --git a/priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs b/priv/repo/migrations/20251019160255_add_llm_api_key_to_courses.exs similarity index 100% rename from priv/repo/migrations/20240320000001_add_llm_api_key_to_courses.exs rename to priv/repo/migrations/20251019160255_add_llm_api_key_to_courses.exs diff --git a/priv/repo/migrations/20250220103623_create_ai_comments.exs b/priv/repo/migrations/20251022103623_create_ai_comments.exs similarity index 100% rename from priv/repo/migrations/20250220103623_create_ai_comments.exs rename to priv/repo/migrations/20251022103623_create_ai_comments.exs diff --git a/priv/repo/migrations/20251007050808_add_llm_assessment_prompt_assessment.exs b/priv/repo/migrations/20251028050808_add_llm_assessment_prompt_assessment.exs similarity index 100% rename from priv/repo/migrations/20251007050808_add_llm_assessment_prompt_assessment.exs rename to priv/repo/migrations/20251028050808_add_llm_assessment_prompt_assessment.exs From f2c02c9d1a70607dceb275c3d90cca15f1dc826f Mon Sep 17 00:00:00 2001 From: Richard Dominick <34370238+RichDom2185@users.noreply.github.com> Date: Tue, 28 Oct 2025 16:04:30 +0800 Subject: [PATCH 56/77] Add newline at EOF --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 80b3e3026..3ab2b5e8c 100644 --- a/.gitignore +++ b/.gitignore @@ -120,4 +120,4 @@ erl_crash.dump /src/source_lexer.erl # Ignore log files -/log \ No newline at end of file +/log From 0d2f0c20f2cf249a9e03aa525bac6333a4d1fa47 Mon Sep 17 00:00:00 2001 From: Richard Dominick <34370238+RichDom2185@users.noreply.github.com> Date: Wed, 29 Oct 2025 13:12:36 +0800 Subject: [PATCH 57/77] Fix indent --- config/dev.secrets.exs.example | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/dev.secrets.exs.example b/config/dev.secrets.exs.example index 9f42df8fc..1ede99148 100644 --- a/config/dev.secrets.exs.example +++ b/config/dev.secrets.exs.example @@ -98,9 +98,9 @@ config :cadet, # ws_endpoint_address: "ws://hostname:port" ] - config :openai, - # TODO: Input your own AES-256 encryption key here for encrypting LLM API keys of 16, 24 or 32 bytes - encryption_key: "" +config :openai, + # TODO: Input your own AES-256 encryption key here for encrypting LLM API keys of 16, 24 or 32 bytes + encryption_key: "" # config :sentry, # dsn: "https://public_key/sentry.io/somethingsomething" From 8cf09b98dc21ac94385ba92d66d90fd6bbcf413d Mon Sep 17 00:00:00 2001 From: Richard Dominick <34370238+RichDom2185@users.noreply.github.com> Date: Wed, 29 Oct 2025 13:20:03 +0800 Subject: [PATCH 58/77] Fix capitalization --- priv/repo/migrations/20251008160219_add_llm_grading_access.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priv/repo/migrations/20251008160219_add_llm_grading_access.exs b/priv/repo/migrations/20251008160219_add_llm_grading_access.exs index c5f337eee..f0f485435 100644 --- a/priv/repo/migrations/20251008160219_add_llm_grading_access.exs +++ b/priv/repo/migrations/20251008160219_add_llm_grading_access.exs @@ -1,4 +1,4 @@ -defmodule Cadet.Repo.Migrations.AddLLMGradingAccess do +defmodule Cadet.Repo.Migrations.AddLlmGradingAccess do use Ecto.Migration def up do From 23541e75d705b3bdf8df38b29757d4a93b7bdc82 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Thu, 30 Oct 2025 16:10:26 +0800 Subject: [PATCH 59/77] Remove llmApiKey from any kind of storage on FE --- lib/cadet_web/views/courses_view.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cadet_web/views/courses_view.ex b/lib/cadet_web/views/courses_view.ex index e41fd22dc..a3a3f443e 100644 --- a/lib/cadet_web/views/courses_view.ex +++ b/lib/cadet_web/views/courses_view.ex @@ -17,7 +17,6 @@ defmodule CadetWeb.CoursesView do enableSourcecast: :enable_sourcecast, enableStories: :enable_stories, enableLlmGrading: :enable_llm_grading, - llmApiKey: :llm_api_key, llmModel: :llm_model, llmApiUrl: :llm_api_url, llmCourseLevelPrompt: :llm_course_level_prompt, From 9ecb1d681599c12c44335bc666a0be17299a6b78 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Thu, 30 Oct 2025 16:27:03 +0800 Subject: [PATCH 60/77] Remove indexes --- priv/repo/migrations/20251022103623_create_ai_comments.exs | 3 --- 1 file changed, 3 deletions(-) diff --git a/priv/repo/migrations/20251022103623_create_ai_comments.exs b/priv/repo/migrations/20251022103623_create_ai_comments.exs index 53ee4f532..7e00d1385 100644 --- a/priv/repo/migrations/20251022103623_create_ai_comments.exs +++ b/priv/repo/migrations/20251022103623_create_ai_comments.exs @@ -13,8 +13,5 @@ defmodule Cadet.Repo.Migrations.CreateAiCommentLogs do add(:final_comment, :text) timestamps() end - - create(index(:ai_comment_logs, [:submission_id])) - create(index(:ai_comment_logs, [:question_id])) end end From e26b33fa6d8a30b3e8c0bd47d903be7a5934c92d Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Thu, 30 Oct 2025 16:27:49 +0800 Subject: [PATCH 61/77] rm todo --- config/dev.secrets.exs.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/dev.secrets.exs.example b/config/dev.secrets.exs.example index 1ede99148..6b142df34 100644 --- a/config/dev.secrets.exs.example +++ b/config/dev.secrets.exs.example @@ -99,7 +99,7 @@ config :cadet, ] config :openai, - # TODO: Input your own AES-256 encryption key here for encrypting LLM API keys of 16, 24 or 32 bytes + # Input your own AES-256 encryption key here for encrypting LLM API keys of 16, 24 or 32 bytes encryption_key: "" # config :sentry, From 0940dd8e84a702db987ad55c2b8c7633b9467a50 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Thu, 30 Oct 2025 16:28:03 +0800 Subject: [PATCH 62/77] rm todo --- config/test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/test.exs b/config/test.exs index 346858849..7e9aa8f86 100644 --- a/config/test.exs +++ b/config/test.exs @@ -102,5 +102,5 @@ config :cadet, Oban, config :cadet, Cadet.Mailer, adapter: Bamboo.TestAdapter config :openai, - # TODO: Input your own AES-256 encryption key here for encrypting LLM API keys + # Input your own AES-256 encryption key here for encrypting LLM API keys encryption_key: "b4u7g0AyN3Tu2br9WSdZQjLMQ8bed/wgQWrH2x3qPdW8D55iv10+ySgs+bxDirWE" From 654c9f4d02bde0ef1699bfe33e7cf9488a451bb3 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Thu, 30 Oct 2025 18:09:06 +0800 Subject: [PATCH 63/77] Re-format ai_comments to reference answer_id instead --- lib/cadet/ai_comments.ex | 38 +-------- lib/cadet/ai_comments/ai_comment.ex | 11 +-- lib/cadet/assessments/answer.ex | 3 +- lib/cadet/assessments/assessments.ex | 73 +++++++++++++---- .../admin_views/admin_grading_view.ex | 1 + .../controllers/generate_ai_comments.ex | 79 +++++++------------ lib/cadet_web/router.ex | 4 +- .../20251022103623_create_ai_comments.exs | 3 +- 8 files changed, 100 insertions(+), 112 deletions(-) diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex index 8e5ce62ac..e37133b98 100644 --- a/lib/cadet/ai_comments.ex +++ b/lib/cadet/ai_comments.ex @@ -26,26 +26,14 @@ defmodule Cadet.AIComments do end end - @doc """ - Retrieves an AI comment for a specific submission and question. - Returns `nil` if no comment exists. - """ - def get_ai_comments_for_submission(submission_id, question_id) do - Repo.one( - from(c in AIComment, - where: c.submission_id == ^submission_id and c.question_id == ^question_id - ) - ) - end - @doc """ Retrieves the latest AI comment for a specific submission and question. Returns `nil` if no comment exists. """ - def get_latest_ai_comment(submission_id, question_id) do + def get_latest_ai_comment(answer_id) do Repo.one( from(c in AIComment, - where: c.submission_id == ^submission_id and c.question_id == ^question_id, + where: c.answer_id == ^answer_id, order_by: [desc: c.inserted_at], limit: 1 ) @@ -56,8 +44,8 @@ defmodule Cadet.AIComments do Updates the final comment for a specific submission and question. Returns the most recent comment entry for that submission/question. """ - def update_final_comment(submission_id, question_id, final_comment) do - comment = get_latest_ai_comment(submission_id, question_id) + def update_final_comment(answer_id, final_comment) do + comment = get_latest_ai_comment(answer_id) case comment do nil -> @@ -86,22 +74,4 @@ defmodule Cadet.AIComments do |> Repo.update() end end - - @doc """ - Updates the chosen comments for a specific submission and question. - Accepts an array of comments and replaces the existing array in the database. - """ - def update_chosen_comments(submission_id, question_id, new_comments) do - comment = get_latest_ai_comment(submission_id, question_id) - - case comment do - nil -> - {:error, :not_found} - - _ -> - comment - |> AIComment.changeset(%{comment_chosen: new_comments}) - |> Repo.update() - end - end end diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex index 3dad339b7..0b5ec3e2d 100644 --- a/lib/cadet/ai_comments/ai_comment.ex +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -14,8 +14,7 @@ defmodule Cadet.AIComments.AIComment do field(:comment_chosen, {:array, :string}) field(:final_comment, :string) - belongs_to(:submission, Cadet.Assessments.Submission) - belongs_to(:question, Cadet.Assessments.Question) + belongs_to(:answer, Cadet.Assessments.Answer) timestamps() end @@ -23,8 +22,7 @@ defmodule Cadet.AIComments.AIComment do def changeset(ai_comment, attrs) do ai_comment |> cast(attrs, [ - :submission_id, - :question_id, + :answer_id, :raw_prompt, :answers_json, :response, @@ -32,8 +30,7 @@ defmodule Cadet.AIComments.AIComment do :comment_chosen, :final_comment ]) - |> validate_required([:submission_id, :question_id, :raw_prompt, :answers_json]) - |> foreign_key_constraint(:submission_id) - |> foreign_key_constraint(:question_id) + |> validate_required([:answer_id, :raw_prompt, :answers_json]) + |> foreign_key_constraint(:answer_id) end end diff --git a/lib/cadet/assessments/answer.ex b/lib/cadet/assessments/answer.ex index cab783a7d..b8f8695b5 100644 --- a/lib/cadet/assessments/answer.ex +++ b/lib/cadet/assessments/answer.ex @@ -10,6 +10,7 @@ defmodule Cadet.Assessments.Answer do alias Cadet.Assessments.Answer.AutogradingStatus alias Cadet.Assessments.AnswerTypes.{MCQAnswer, ProgrammingAnswer, VotingAnswer} alias Cadet.Assessments.{Question, QuestionType, Submission} + alias Cadet.AIComments.AIComment @type t :: %__MODULE__{} @@ -29,7 +30,7 @@ defmodule Cadet.Assessments.Answer do belongs_to(:grader, CourseRegistration) belongs_to(:submission, Submission) belongs_to(:question, Question) - has_many(:ai_comments, through: [:submission, :ai_comments]) + has_many(:ai_comments, AIComment, on_delete: :delete_all) timestamps() end diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 3e4dc1f99..3c0d69331 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -3043,10 +3043,60 @@ defmodule Cadet.Assessments do } end +@spec get_answer(integer() | String.t()) :: + {:ok, Answer.t()} | {:error, {:bad_request, String.t()}} +def get_answer(id) when is_ecto_id(id) do + answer = + Answer + |> where(id: ^id) + # [a] are bindings (in SQL it is similar to FROM answers "AS a"), + # this line's alias is INNER JOIN ... "AS q" + |> join(:inner, [a], q in assoc(a, :question)) + |> join(:inner, [_, q], ast in assoc(q, :assessment)) + |> join(:inner, [..., ast], ac in assoc(ast, :config)) + |> join(:left, [a, ...], g in assoc(a, :grader)) + |> join(:left, [_, ..., g], gu in assoc(g, :user)) + |> join(:inner, [a, ...], s in assoc(a, :submission)) + |> join(:left, [_, ..., s], st in assoc(s, :student)) + |> join(:left, [..., st], u in assoc(st, :user)) + |> join(:left, [..., s, _, _], t in assoc(s, :team)) + |> join(:left, [..., t], tm in assoc(t, :team_members)) + |> join(:left, [..., tm], tms in assoc(tm, :student)) + |> join(:left, [..., tms], tmu in assoc(tms, :user)) + |> join(:left, [a, ...], ai in assoc(a, :ai_comments)) + |> preload([_, q, ast, ac, g, gu, s, st, u, t, tm, tms, tmu, ai], + ai_comments: ai, + question: {q, assessment: {ast, config: ac}}, + grader: {g, user: gu}, + submission: + {s, student: {st, user: u}, team: {t, team_members: {tm, student: {tms, user: tmu}}}} + ) + |> Repo.one() + + if is_nil(answer) do + {:error, {:bad_request, "Answer not found."}} + else + + + if answer.question.type == :voting do + empty_contest_entries = Map.put(answer.question.question, :contest_entries, []) + empty_popular_leaderboard = Map.put(empty_contest_entries, :popular_leaderboard, []) + empty_contest_leaderboard = Map.put(empty_popular_leaderboard, :contest_leaderboard, []) + question = Map.put(answer.question, :question, empty_contest_leaderboard) + Map.put(answer, :question, question) + end + + {:ok, answer} + end + + + +end + @spec get_answers_in_submission(integer() | String.t()) :: {:ok, {[Answer.t()], Assessment.t()}} | {:error, {:bad_request, String.t()}} - def get_answers_in_submission(id, question_id \\ nil) when is_ecto_id(id) do + def get_answers_in_submission(id) when is_ecto_id(id) do base_query = Answer |> where(submission_id: ^id) @@ -3058,29 +3108,23 @@ defmodule Cadet.Assessments do |> join(:left, [a, ...], g in assoc(a, :grader)) |> join(:left, [_, ..., g], gu in assoc(g, :user)) |> join(:inner, [a, ...], s in assoc(a, :submission)) - |> join(:left, [..., s], ai in assoc(s, :ai_comments)) - |> join(:left, [_, ..., s, _], st in assoc(s, :student)) + |> join(:left, [_, ..., s], st in assoc(s, :student)) |> join(:left, [..., st], u in assoc(st, :user)) - |> join(:left, [..., s, _, _, _], t in assoc(s, :team)) + |> join(:left, [..., s, _, _], t in assoc(s, :team)) |> join(:left, [..., t], tm in assoc(t, :team_members)) |> join(:left, [..., tm], tms in assoc(tm, :student)) |> join(:left, [..., tms], tmu in assoc(tms, :user)) - |> preload([_, q, ast, ac, g, gu, s, ai, st, u, t, tm, tms, tmu], - ai_comments: ai, + |> join(:left, [a, ...], ai in assoc(a, :ai_comments)) + |> preload([_, q, ast, ac, g, gu, s, st, u, t, tm, tms, tmu, ai], question: {q, assessment: {ast, config: ac}}, grader: {g, user: gu}, submission: - {s, student: {st, user: u}, team: {t, team_members: {tm, student: {tms, user: tmu}}}} + {s, student: {st, user: u}, team: {t, team_members: {tm, student: {tms, user: tmu}}}}, + ai_comments: ai ) - answer_query = - case question_id do - nil -> base_query - _ -> base_query |> where(question_id: ^question_id) - end - answers = - answer_query + base_query |> Repo.all() |> Enum.sort_by(& &1.question.display_order) |> Enum.map(fn ans -> @@ -3091,6 +3135,7 @@ defmodule Cadet.Assessments do question = Map.put(ans.question, :question, empty_contest_leaderboard) Map.put(ans, :question, question) else + ans end end) diff --git a/lib/cadet_web/admin_views/admin_grading_view.ex b/lib/cadet_web/admin_views/admin_grading_view.ex index 3521a1e01..71fbfc987 100644 --- a/lib/cadet_web/admin_views/admin_grading_view.ex +++ b/lib/cadet_web/admin_views/admin_grading_view.ex @@ -151,6 +151,7 @@ defmodule CadetWeb.AdminGradingView do def render("grading_info.json", %{answer: answer}) do transform_map_for_view(answer, %{ + id: &(&1.id), ai_comments: &extract_ai_comments_per_answer(&1.question_id, &1.ai_comments), student: &extract_student_data(&1.submission.student), team: &extract_team_data(&1.submission.team), diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 764eeb7f9..2a3967945 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -7,19 +7,18 @@ defmodule CadetWeb.AICodeAnalysisController do alias Cadet.{Assessments, AIComments, Courses} # For logging outputs to both database and file - defp save_comment(submission_id, question_id, raw_prompt, answers_json, response, error \\ nil) do + defp save_comment(answer_id, raw_prompt, answers_json, response, error \\ nil) do # Log to database attrs = %{ - submission_id: submission_id, - question_id: question_id, + answer_id: answer_id, raw_prompt: raw_prompt, answers_json: answers_json, response: response, error: error } - # Check if a comment already exists for the given submission_id and question_id - case AIComments.get_latest_ai_comment(submission_id, question_id) do + # Check if a comment already exists for the given answer_id + case AIComments.get_latest_ai_comment(answer_id) do nil -> # If no existing comment, create a new one case AIComments.create_ai_comment(attrs) do @@ -75,15 +74,14 @@ defmodule CadetWeb.AICodeAnalysisController do end @doc """ - Fetches the question details and answers based on submissionid and questionid and generates AI-generated comments. + Fetches the question details and answers based on answer_id and generates AI-generated comments. """ def generate_ai_comments(conn, %{ - "submissionid" => submission_id, - "questionid" => question_id, + "answer_id" => answer_id, "course_id" => course_id }) - when is_ecto_id(submission_id) do - with {qid, ""} <- Integer.parse(question_id), + when is_ecto_id(answer_id) do + with {answer_id_parsed, ""} <- Integer.parse(answer_id), {:ok, course} <- Courses.get_course_config(course_id), {:ok} <- ensure_llm_enabled(course), {:ok, key} <- decrypt_llm_api_key(course.llm_api_key), @@ -94,27 +92,25 @@ defmodule CadetWeb.AICodeAnalysisController do course.llm_api_url, course.llm_course_level_prompt ), - {:ok, {answers, _}} <- Assessments.get_answers_in_submission(submission_id, qid) do + {:ok, answer} <- Assessments.get_answer(answer_id_parsed) do # Get head of answers (should only be one answer for given submission # and question since we filter to only 1 question) - case answers do - [] -> + case answer do + nil -> conn |> put_status(:not_found) - |> text("No answer found for the given submission and question_id") + |> text("No answer found for the given answer_id") _ -> analyze_code( conn, %{ - answer: hd(answers), - submission_id: submission_id, - question_id: qid, + answer: answer, api_key: key, llm_model: course.llm_model, llm_api_url: course.llm_api_url, course_prompt: course.llm_course_level_prompt, - assessment_prompt: Assessments.get_llm_assessment_prompt(qid) + assessment_prompt: Assessments.get_llm_assessment_prompt(answer.question_id) } ) end @@ -142,20 +138,6 @@ defmodule CadetWeb.AICodeAnalysisController do end end - defp transform_answers(answers) do - Enum.map(answers, fn answer -> - %{ - id: answer.id, - comments: answer.comments, - autograding_status: answer.autograding_status, - autograding_results: answer.autograding_results, - code: answer.answer["code"], - question_id: answer.question_id, - question_content: answer.question["content"] - } - end) - end - defp format_student_answer(answer) do """ **Student Answer:** @@ -212,8 +194,6 @@ defmodule CadetWeb.AICodeAnalysisController do conn, %{ answer: answer, - submission_id: submission_id, - question_id: question_id, api_key: api_key, llm_model: llm_model, llm_api_url: llm_api_url, @@ -247,7 +227,7 @@ defmodule CadetWeb.AICodeAnalysisController do {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> case Jason.decode(body) do {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> - save_comment(submission_id, question_id, system_prompt, formatted_answer, response) + save_comment(answer.id, system_prompt, formatted_answer, response) comments_list = String.split(response, "|||") filtered_comments = @@ -259,8 +239,7 @@ defmodule CadetWeb.AICodeAnalysisController do {:ok, other} -> save_comment( - submission_id, - question_id, + answer.id, system_prompt, formatted_answer, Jason.encode!(other), @@ -273,8 +252,7 @@ defmodule CadetWeb.AICodeAnalysisController do {:error, err} -> save_comment( - submission_id, - question_id, + answer.id, system_prompt, formatted_answer, nil, @@ -288,8 +266,7 @@ defmodule CadetWeb.AICodeAnalysisController do {:ok, %HTTPoison.Response{status_code: status, body: body}} -> save_comment( - submission_id, - question_id, + answer.id, system_prompt, formatted_answer, nil, @@ -301,7 +278,7 @@ defmodule CadetWeb.AICodeAnalysisController do |> text("API request failed with status #{status}: #{body}") {:error, %HTTPoison.Error{reason: reason}} -> - save_comment(submission_id, question_id, system_prompt, formatted_answer, nil, reason) + save_comment(answer.id, system_prompt, formatted_answer, nil, reason) conn |> put_status(:internal_server_error) @@ -313,11 +290,10 @@ defmodule CadetWeb.AICodeAnalysisController do Saves the final comment chosen for a submission. """ def save_final_comment(conn, %{ - "submissionid" => submission_id, - "questionid" => question_id, + "answer_id" => answer_id, "comment" => comment }) do - case AIComments.update_final_comment(submission_id, question_id, comment) do + case AIComments.update_final_comment(answer_id, comment) do {:ok, _updated_comment} -> json(conn, %{"status" => "success"}) @@ -329,7 +305,7 @@ defmodule CadetWeb.AICodeAnalysisController do end swagger_path :generate_ai_comments do - post("/courses/{courseId}/admin/generate-comments/{submissionId}/{questionId}") + post("/courses/{course_id}/admin/generate-comments/{answer_id}") summary("Generate AI comments for a given submission.") @@ -339,9 +315,8 @@ defmodule CadetWeb.AICodeAnalysisController do produces("application/json") parameters do - courseId(:path, :integer, "course id", required: true) - submissionId(:path, :integer, "submission id", required: true) - questionId(:path, :integer, "question id", required: true) + course_id(:path, :integer, "course id", required: true) + answer_id(:path, :integer, "answer id", required: true) end response(200, "OK", Schema.ref(:GenerateAIComments)) @@ -352,7 +327,7 @@ defmodule CadetWeb.AICodeAnalysisController do end swagger_path :save_final_comment do - post("/courses/{courseId}/admin/save-final-comment/{submissionId}/{questionId}") + post("/courses/{course_id}/admin/save-final-comment/{answer_id}") summary("Save the final comment chosen for a submission.") @@ -362,8 +337,8 @@ defmodule CadetWeb.AICodeAnalysisController do produces("application/json") parameters do - submissionId(:path, :integer, "submission id", required: true) - questionId(:path, :integer, "question id", required: true) + course_id(:path, :integer, "course id", required: true) + answer_id(:path, :integer, "answer id", required: true) comment(:body, :string, "The final comment to save", required: true) end diff --git a/lib/cadet_web/router.ex b/lib/cadet_web/router.ex index 0bef08e14..8c917c918 100644 --- a/lib/cadet_web/router.ex +++ b/lib/cadet_web/router.ex @@ -221,7 +221,7 @@ defmodule CadetWeb.Router do post("/grading/:submissionid/:questionid", AdminGradingController, :update) post( - "/generate-comments/:submissionid/:questionid", + "/generate-comments/:answer_id", AICodeAnalysisController, :generate_ai_comments ) @@ -233,7 +233,7 @@ defmodule CadetWeb.Router do ) post( - "/save-final-comment/:submissionid/:questionid", + "/save-final-comment/:answer_id", AICodeAnalysisController, :save_final_comment ) diff --git a/priv/repo/migrations/20251022103623_create_ai_comments.exs b/priv/repo/migrations/20251022103623_create_ai_comments.exs index 7e00d1385..425fb4b16 100644 --- a/priv/repo/migrations/20251022103623_create_ai_comments.exs +++ b/priv/repo/migrations/20251022103623_create_ai_comments.exs @@ -3,8 +3,7 @@ defmodule Cadet.Repo.Migrations.CreateAiCommentLogs do def change do create table(:ai_comment_logs) do - add(:submission_id, references(:submissions, on_delete: :delete_all), null: false) - add(:question_id, references(:questions, on_delete: :delete_all), null: false) + add(:answer_id, references(:answers, on_delete: :delete_all), null: false) add(:raw_prompt, :text, null: false) add(:answers_json, :text, null: false) add(:response, :text) From af395dfaa90d2f530043243432b18609399b86e5 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Thu, 30 Oct 2025 18:28:16 +0800 Subject: [PATCH 64/77] Abstract out + remove un-used field --- lib/cadet/ai_comments/ai_comment.ex | 2 - lib/cadet/courses/course.ex | 29 +------- .../controllers/generate_ai_comments.ex | 66 ++++++------------- lib/cadet_web/helpers/ai_comments_helpers.ex | 64 ++++++++++++++++++ .../20251022103623_create_ai_comments.exs | 1 - 5 files changed, 85 insertions(+), 77 deletions(-) create mode 100644 lib/cadet_web/helpers/ai_comments_helpers.ex diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex index 0b5ec3e2d..f12fde508 100644 --- a/lib/cadet/ai_comments/ai_comment.ex +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -11,7 +11,6 @@ defmodule Cadet.AIComments.AIComment do field(:answers_json, :string) field(:response, :string) field(:error, :string) - field(:comment_chosen, {:array, :string}) field(:final_comment, :string) belongs_to(:answer, Cadet.Assessments.Answer) @@ -27,7 +26,6 @@ defmodule Cadet.AIComments.AIComment do :answers_json, :response, :error, - :comment_chosen, :final_comment ]) |> validate_required([:answer_id, :raw_prompt, :answers_json]) diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index a14e87a73..d3be44f7b 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -5,6 +5,7 @@ defmodule Cadet.Courses.Course do use Cadet, :model alias Cadet.Courses.AssessmentConfig + alias Cadet.AICommentsHelpers @type t :: %__MODULE__{ course_name: String.t(), @@ -70,36 +71,10 @@ defmodule Cadet.Courses.Course do |> put_encrypted_llm_api_key() end - def encrypt_llm_api_key(llm_api_key) do - secret = Application.get_env(:openai, :encryption_key) - - if is_binary(secret) and byte_size(secret) >= 16 do - # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256 - key = binary_part(secret, 0, min(32, byte_size(secret))) - # Use AES in GCM mode for encryption - iv = :crypto.strong_rand_bytes(16) - - {ciphertext, tag} = - :crypto.crypto_one_time_aead( - :aes_gcm, - key, - iv, - llm_api_key, - "", - true - ) - - # Store both the IV, ciphertext and tag - encrypted = Base.encode64(iv <> tag <> ciphertext) - else - {:error, :invalid_encryption_key} - end - end - def put_encrypted_llm_api_key(changeset) do if llm_api_key = get_change(changeset, :llm_api_key) do if is_binary(llm_api_key) and llm_api_key != "" do - encrypted = encrypt_llm_api_key(llm_api_key) + encrypted = AICommentsHelpers.encrypt_llm_api_key(llm_api_key) case encrypted do {:error, :invalid_encryption_key} -> diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 2a3967945..fab2fbbdc 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -5,6 +5,8 @@ defmodule CadetWeb.AICodeAnalysisController do require Logger alias Cadet.{Assessments, AIComments, Courses} + alias CadetWeb.AICodeAnalysisController + alias CadetWeb.AICommentsHelpers # For logging outputs to both database and file defp save_comment(answer_id, raw_prompt, answers_json, response, error \\ nil) do @@ -84,7 +86,7 @@ defmodule CadetWeb.AICodeAnalysisController do with {answer_id_parsed, ""} <- Integer.parse(answer_id), {:ok, course} <- Courses.get_course_config(course_id), {:ok} <- ensure_llm_enabled(course), - {:ok, key} <- decrypt_llm_api_key(course.llm_api_key), + {:ok, key} <- AICommentsHelpers.decrypt_llm_api_key(course.llm_api_key), {:ok} <- check_llm_grading_parameters( key, @@ -123,7 +125,7 @@ defmodule CadetWeb.AICodeAnalysisController do {:decrypt_error, err} -> conn |> put_status(:internal_server_error) - |> text("Failed to decrypt LLM API key: #{inspect(err)}") + |> text("Failed to decrypt LLM API key") # Errors for check_llm_grading_parameters {:parameter_error, error_msg} -> @@ -152,25 +154,25 @@ defmodule CadetWeb.AICodeAnalysisController do "\n\n" <> (assessment_prompt || "") <> "\n\n" <> - """ - **Additional Instructions for this Question:** - #{answer.question.question["llm_prompt"] || "N/A"} +""" +**Additional Instructions for this Question:** +#{answer.question.question["llm_prompt"] || "N/A"} - **Question:** - ``` - #{answer.question.question["content"] || "N/A"} - ``` +**Question:** +``` +#{answer.question.question["content"] || "N/A"} +``` - **Model Solution:** - ``` - #{answer.question.question["solution"] || "N/A"} - ``` +**Model Solution:** +``` +#{answer.question.question["solution"] || "N/A"} +``` - **Autograding Status:** #{answer.autograding_status || "N/A"} - **Autograding Results:** #{format_autograding_results(answer.autograding_results)} +**Autograding Status:** #{answer.autograding_status || "N/A"} +**Autograding Results:** #{format_autograding_results(answer.autograding_results)} - The student answer will be given below as part of the User Prompt. - """ +The student answer will be given below as part of the User Prompt. +""" end defp format_autograding_results(nil), do: "N/A" @@ -365,35 +367,5 @@ defmodule CadetWeb.AICodeAnalysisController do } end - defp decrypt_llm_api_key(nil), do: nil - defp decrypt_llm_api_key(encrypted_key) do - case Application.get_env(:openai, :encryption_key) do - secret when is_binary(secret) and byte_size(secret) >= 16 -> - key = binary_part(secret, 0, min(32, byte_size(secret))) - - case Base.decode64(encrypted_key) do - {:ok, decoded} -> - iv = binary_part(decoded, 0, 16) - tag = binary_part(decoded, 16, 16) - ciphertext = binary_part(decoded, 32, byte_size(decoded) - 32) - - case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do - plain_text when is_binary(plain_text) -> {:ok, plain_text} - _ -> {:decrypt_error, :decryption_failed} - end - - _ -> - Logger.error( - "Failed to decode encrypted key, is it a valid AES-256 key of 16, 24 or 32 bytes?" - ) - - {:decrypt_error, :decryption_failed} - end - - _ -> - Logger.error("Encryption key not configured") - {:decrypt_error, :invalid_encryption_key} - end - end end diff --git a/lib/cadet_web/helpers/ai_comments_helpers.ex b/lib/cadet_web/helpers/ai_comments_helpers.ex new file mode 100644 index 000000000..23440cf9e --- /dev/null +++ b/lib/cadet_web/helpers/ai_comments_helpers.ex @@ -0,0 +1,64 @@ +defmodule CadetWeb.AICommentsHelpers do + @moduledoc """ + Helper functions for Managing LLM related logic + """ + + def decrypt_llm_api_key(nil), do: nil + + def decrypt_llm_api_key(encrypted_key) do + case Application.get_env(:openai, :encryption_key) do + secret when is_binary(secret) and byte_size(secret) >= 16 -> + key = binary_part(secret, 0, min(32, byte_size(secret))) + + case Base.decode64(encrypted_key) do + {:ok, decoded} -> + iv = binary_part(decoded, 0, 16) + tag = binary_part(decoded, 16, 16) + ciphertext = binary_part(decoded, 32, byte_size(decoded) - 32) + + case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do + plain_text when is_binary(plain_text) -> {:ok, plain_text} + _ -> {:decrypt_error, :decryption_failed} + end + + _ -> + Logger.error( + "Failed to decode encrypted key, is it a valid AES-256 key of 16, 24 or 32 bytes?" + ) + + {:decrypt_error, :decryption_failed} + end + + _ -> + Logger.error("Encryption key not configured") + {:decrypt_error, :invalid_encryption_key} + end + end + + def encrypt_llm_api_key(llm_api_key) do + secret = Application.get_env(:openai, :encryption_key) + + if is_binary(secret) and byte_size(secret) >= 16 do + # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256 + key = binary_part(secret, 0, min(32, byte_size(secret))) + # Use AES in GCM mode for encryption + iv = :crypto.strong_rand_bytes(16) + + {ciphertext, tag} = + :crypto.crypto_one_time_aead( + :aes_gcm, + key, + iv, + llm_api_key, + "", + true + ) + + # Store both the IV, ciphertext and tag + encrypted = Base.encode64(iv <> tag <> ciphertext) + else + {:error, :invalid_encryption_key} + end + end + +end diff --git a/priv/repo/migrations/20251022103623_create_ai_comments.exs b/priv/repo/migrations/20251022103623_create_ai_comments.exs index 425fb4b16..5093d1a43 100644 --- a/priv/repo/migrations/20251022103623_create_ai_comments.exs +++ b/priv/repo/migrations/20251022103623_create_ai_comments.exs @@ -8,7 +8,6 @@ defmodule Cadet.Repo.Migrations.CreateAiCommentLogs do add(:answers_json, :text, null: false) add(:response, :text) add(:error, :text) - add(:comment_chosen, {:array, :text}) add(:final_comment, :text) timestamps() end From f1b8154f1e87c40f8f7746f52b2866b4bff5a354 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Thu, 30 Oct 2025 18:43:42 +0800 Subject: [PATCH 65/77] Add delimeter + bug fixes --- lib/cadet/assessments/assessments.ex | 1 - lib/cadet/courses/course.ex | 2 +- lib/cadet_web/admin_views/admin_grading_view.ex | 6 +++--- lib/cadet_web/helpers/ai_comments_helpers.ex | 17 ++++++++++------- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 3c0d69331..1a41221ba 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -3135,7 +3135,6 @@ end question = Map.put(ans.question, :question, empty_contest_leaderboard) Map.put(ans, :question, question) else - ans end end) diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index d3be44f7b..b8a113be1 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -5,7 +5,7 @@ defmodule Cadet.Courses.Course do use Cadet, :model alias Cadet.Courses.AssessmentConfig - alias Cadet.AICommentsHelpers + alias CadetWeb.AICommentsHelpers @type t :: %__MODULE__{ course_name: String.t(), diff --git a/lib/cadet_web/admin_views/admin_grading_view.ex b/lib/cadet_web/admin_views/admin_grading_view.ex index 71fbfc987..354105262 100644 --- a/lib/cadet_web/admin_views/admin_grading_view.ex +++ b/lib/cadet_web/admin_views/admin_grading_view.ex @@ -152,7 +152,7 @@ defmodule CadetWeb.AdminGradingView do def render("grading_info.json", %{answer: answer}) do transform_map_for_view(answer, %{ id: &(&1.id), - ai_comments: &extract_ai_comments_per_answer(&1.question_id, &1.ai_comments), + ai_comments: &extract_ai_comments_per_answer(&1.id, &1.ai_comments), student: &extract_student_data(&1.submission.student), team: &extract_team_data(&1.submission.team), question: &build_grading_question/1, @@ -165,11 +165,11 @@ defmodule CadetWeb.AdminGradingView do %{cols: cols, rows: summary} end - defp extract_ai_comments_per_answer(question_id, ai_comments) do + defp extract_ai_comments_per_answer(id, ai_comments) do matching_comment = ai_comments # Equivalent to fn comment -> comment.question_id == question_id end - |> Enum.find(&(&1.question_id == question_id)) + |> Enum.find(&(&1.answer_id == id)) case matching_comment do nil -> nil diff --git a/lib/cadet_web/helpers/ai_comments_helpers.ex b/lib/cadet_web/helpers/ai_comments_helpers.ex index 23440cf9e..80336f586 100644 --- a/lib/cadet_web/helpers/ai_comments_helpers.ex +++ b/lib/cadet_web/helpers/ai_comments_helpers.ex @@ -12,15 +12,18 @@ defmodule CadetWeb.AICommentsHelpers do case Base.decode64(encrypted_key) do {:ok, decoded} -> - iv = binary_part(decoded, 0, 16) - tag = binary_part(decoded, 16, 16) - ciphertext = binary_part(decoded, 32, byte_size(decoded) - 32) - case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do - plain_text when is_binary(plain_text) -> {:ok, plain_text} - _ -> {:decrypt_error, :decryption_failed} + with [iv, tag, ciphertext] <- :binary.split(decoded, <<"|">>, [:global]) do + case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do + plain_text when is_binary(plain_text) -> {:ok, plain_text} + _ -> {:decrypt_error, :decryption_failed} + end + else + _ -> {:error, :invalid_format} end + + _ -> Logger.error( "Failed to decode encrypted key, is it a valid AES-256 key of 16, 24 or 32 bytes?" @@ -55,7 +58,7 @@ defmodule CadetWeb.AICommentsHelpers do ) # Store both the IV, ciphertext and tag - encrypted = Base.encode64(iv <> tag <> ciphertext) + encrypted = Base.encode64(iv <> "|" <> tag <> "|" <> ciphertext) else {:error, :invalid_encryption_key} end From 3450096c06266175043549d8f9cf625e31852170 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Thu, 30 Oct 2025 20:44:45 +0800 Subject: [PATCH 66/77] Mix format --- lib/cadet/assessments/assessments.ex | 15 +-- .../admin_views/admin_grading_view.ex | 2 +- .../controllers/generate_ai_comments.ex | 108 +++++------------- lib/cadet_web/helpers/ai_comments_helpers.ex | 4 - 4 files changed, 36 insertions(+), 93 deletions(-) diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 1a41221ba..ed0cbfac5 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -3043,10 +3043,10 @@ defmodule Cadet.Assessments do } end -@spec get_answer(integer() | String.t()) :: - {:ok, Answer.t()} | {:error, {:bad_request, String.t()}} -def get_answer(id) when is_ecto_id(id) do - answer = + @spec get_answer(integer() | String.t()) :: + {:ok, Answer.t()} | {:error, {:bad_request, String.t()}} + def get_answer(id) when is_ecto_id(id) do + answer = Answer |> where(id: ^id) # [a] are bindings (in SQL it is similar to FROM answers "AS a"), @@ -3076,8 +3076,6 @@ def get_answer(id) when is_ecto_id(id) do if is_nil(answer) do {:error, {:bad_request, "Answer not found."}} else - - if answer.question.type == :voting do empty_contest_entries = Map.put(answer.question.question, :contest_entries, []) empty_popular_leaderboard = Map.put(empty_contest_entries, :popular_leaderboard, []) @@ -3088,10 +3086,7 @@ def get_answer(id) when is_ecto_id(id) do {:ok, answer} end - - - -end + end @spec get_answers_in_submission(integer() | String.t()) :: {:ok, {[Answer.t()], Assessment.t()}} diff --git a/lib/cadet_web/admin_views/admin_grading_view.ex b/lib/cadet_web/admin_views/admin_grading_view.ex index 354105262..fc4be7ba3 100644 --- a/lib/cadet_web/admin_views/admin_grading_view.ex +++ b/lib/cadet_web/admin_views/admin_grading_view.ex @@ -151,7 +151,7 @@ defmodule CadetWeb.AdminGradingView do def render("grading_info.json", %{answer: answer}) do transform_map_for_view(answer, %{ - id: &(&1.id), + id: & &1.id, ai_comments: &extract_ai_comments_per_answer(&1.id, &1.ai_comments), student: &extract_student_data(&1.submission.student), team: &extract_team_data(&1.submission.team), diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index fab2fbbdc..2dca80513 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -154,25 +154,25 @@ defmodule CadetWeb.AICodeAnalysisController do "\n\n" <> (assessment_prompt || "") <> "\n\n" <> -""" -**Additional Instructions for this Question:** -#{answer.question.question["llm_prompt"] || "N/A"} + """ + **Additional Instructions for this Question:** + #{answer.question.question["llm_prompt"] || "N/A"} -**Question:** -``` -#{answer.question.question["content"] || "N/A"} -``` + **Question:** + ``` + #{answer.question.question["content"] || "N/A"} + ``` -**Model Solution:** -``` -#{answer.question.question["solution"] || "N/A"} -``` + **Model Solution:** + ``` + #{answer.question.question["solution"] || "N/A"} + ``` -**Autograding Status:** #{answer.autograding_status || "N/A"} -**Autograding Results:** #{format_autograding_results(answer.autograding_results)} + **Autograding Status:** #{answer.autograding_status || "N/A"} + **Autograding Results:** #{format_autograding_results(answer.autograding_results)} -The student answer will be given below as part of the User Prompt. -""" + The student answer will be given below as part of the User Prompt. + """ end defp format_autograding_results(nil), do: "N/A" @@ -211,80 +211,34 @@ The student answer will be given below as part of the User Prompt. system_prompt = format_system_prompt(course_prompt, assessment_prompt, answer) # Combine prompts if llm_prompt exists input = - %{ + [ model: llm_model, messages: [ %{role: "system", content: system_prompt}, %{role: "user", content: formatted_answer} ] - } - |> Jason.encode!() + ] - headers = [ - {"Authorization", "Bearer #{api_key}"}, - {"Content-Type", "application/json"} - ] - - case call_llm_endpoint(llm_api_url, input, headers) do - {:ok, %HTTPoison.Response{status_code: 200, body: body}} -> - case Jason.decode(body) do - {:ok, %{"choices" => [%{"message" => %{"content" => response}}]}} -> - save_comment(answer.id, system_prompt, formatted_answer, response) - comments_list = String.split(response, "|||") - - filtered_comments = - Enum.filter(comments_list, fn comment -> - String.trim(comment) != "" - end) - - json(conn, %{"comments" => filtered_comments}) - - {:ok, other} -> - save_comment( - answer.id, - system_prompt, - formatted_answer, - Jason.encode!(other), - "Unexpected JSON shape" - ) - - conn - |> put_status(:bad_gateway) - |> text("Unexpected response format from OpenAI API") - - {:error, err} -> - save_comment( - answer.id, - system_prompt, - formatted_answer, - nil, - "Failed to parse response from OpenAI API" - ) - - conn - |> put_status(:internal_server_error) - |> text("Failed to parse response from OpenAI API") - end + case OpenAI.chat_completion(input, %OpenAI.Config{ + base_url: llm_api_url, + api_key: api_key + }) do + {:ok, %{"choices" => [%{"message" => %{"content" => content}} | _]}} -> + {:ok, content} - {:ok, %HTTPoison.Response{status_code: status, body: body}} -> - save_comment( - answer.id, - system_prompt, - formatted_answer, - nil, - "API request failed with status #{status}" - ) + {:ok, other} -> + save_comment(answer.id, system_prompt, formatted_answer, other, "Unexpected JSON shape") conn - |> put_status(:internal_server_error) - |> text("API request failed with status #{status}: #{body}") + |> put_status(:bad_gateway) + |> text("Unexpected response format from LLM") - {:error, %HTTPoison.Error{reason: reason}} -> - save_comment(answer.id, system_prompt, formatted_answer, nil, reason) + {:error, reason} -> + save_comment(answer.id, system_prompt, formatted_answer, nil, inspect(reason)) conn |> put_status(:internal_server_error) - |> text("HTTP request error: #{inspect(reason)}") + |> text("LLM request error: #{inspect(reason)}") end end @@ -366,6 +320,4 @@ The student answer will be given below as part of the User Prompt. end } end - - end diff --git a/lib/cadet_web/helpers/ai_comments_helpers.ex b/lib/cadet_web/helpers/ai_comments_helpers.ex index 80336f586..da9d95084 100644 --- a/lib/cadet_web/helpers/ai_comments_helpers.ex +++ b/lib/cadet_web/helpers/ai_comments_helpers.ex @@ -12,7 +12,6 @@ defmodule CadetWeb.AICommentsHelpers do case Base.decode64(encrypted_key) do {:ok, decoded} -> - with [iv, tag, ciphertext] <- :binary.split(decoded, <<"|">>, [:global]) do case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do plain_text when is_binary(plain_text) -> {:ok, plain_text} @@ -22,8 +21,6 @@ defmodule CadetWeb.AICommentsHelpers do _ -> {:error, :invalid_format} end - - _ -> Logger.error( "Failed to decode encrypted key, is it a valid AES-256 key of 16, 24 or 32 bytes?" @@ -63,5 +60,4 @@ defmodule CadetWeb.AICommentsHelpers do {:error, :invalid_encryption_key} end end - end From d02383d529e0a70f04a033dac9b87a3456d14dad Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Thu, 30 Oct 2025 21:20:18 +0800 Subject: [PATCH 67/77] Switch to openAI module --- .../controllers/generate_ai_comments.ex | 20 +++++++++++++++---- mix.exs | 6 +++--- mix.lock | 10 +++++++--- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 2dca80513..5ccefcbf5 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -220,11 +220,23 @@ defmodule CadetWeb.AICodeAnalysisController do ] case OpenAI.chat_completion(input, %OpenAI.Config{ - base_url: llm_api_url, - api_key: api_key + api_url: llm_api_url, + api_key: api_key, + http_options: [ + timeout: 60_000, # connect timeout + recv_timeout: 60_000 # response timeout + ] }) do - {:ok, %{"choices" => [%{"message" => %{"content" => content}} | _]}} -> - {:ok, content} + {:ok, %{choices: [%{"message" => %{"content" => content}} | _]}} -> + save_comment(answer.id, system_prompt, formatted_answer, content) + comments_list = String.split(content, "|||") + + filtered_comments = + Enum.filter(comments_list, fn comment -> + String.trim(comment) != "" + end) + + json(conn, %{"comments" => filtered_comments}) {:ok, other} -> save_comment(answer.id, system_prompt, formatted_answer, other, "Unexpected JSON shape") diff --git a/mix.exs b/mix.exs index c76c48082..f28c28ce4 100644 --- a/mix.exs +++ b/mix.exs @@ -67,10 +67,10 @@ defmodule Cadet.Mixfile do {:guardian, "~> 2.0"}, {:guardian_db, "~> 2.0"}, {:hackney, "~> 1.6"}, - {:httpoison, "~> 1.6"}, + {:httpoison, "~> 2.2.3"}, {:jason, "~> 1.2"}, - {:openai, "~> 0.4.1"}, - {:openid_connect, "~> 0.2"}, + {:openai, "~> 0.6.2"}, + {:openid_connect, "~> 1.0.0"}, {:phoenix, "~> 1.5"}, {:phoenix_view, "~> 2.0"}, {:phoenix_ecto, "~> 4.0"}, diff --git a/mix.lock b/mix.lock index 1b5b1828b..b3c498bc7 100644 --- a/mix.lock +++ b/mix.lock @@ -47,6 +47,7 @@ "exvcr": {:hex, :exvcr, "0.16.0", "11579f43c88ae81f57c82ce4f09e3ebda4c40117c859ed39e61a653c3a0b4ff4", [:mix], [{:exjsx, "~> 4.0", [hex: :exjsx, repo: "hexpm", optional: false]}, {:finch, "~> 0.16", [hex: :finch, repo: "hexpm", optional: true]}, {:httpoison, "~> 1.0 or ~> 2.0", [hex: :httpoison, repo: "hexpm", optional: true]}, {:httpotion, "~> 3.1", [hex: :httpotion, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:meck, "~> 0.9", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "8f576af22369942f7a1482baff1f31e2f45983cf6fac45d49d2bd2e84b4d5be8"}, "faker": {:hex, :faker, "0.18.0", "943e479319a22ea4e8e39e8e076b81c02827d9302f3d32726c5bf82f430e6e14", [:mix], [], "hexpm", "bfbdd83958d78e2788e99ec9317c4816e651ad05e24cfd1196ce5db5b3e81797"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, + "finch": {:hex, :finch, "0.20.0", "5330aefb6b010f424dcbbc4615d914e9e3deae40095e73ab0c1bb0968933cadf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2658131a74d051aabfcba936093c903b8e89da9a1b63e430bee62045fa9b2ee2"}, "gen_smtp": {:hex, :gen_smtp, "1.3.0", "62c3d91f0dcf6ce9db71bcb6881d7ad0d1d834c7f38c13fa8e952f4104a8442e", [:rebar3], [{:ranch, ">= 1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "0b73fbf069864ecbce02fe653b16d3f35fd889d0fdd4e14527675565c39d84e6"}, "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"}, "gettext": {:hex, :gettext, "0.26.2", "5978aa7b21fada6deabf1f6341ddba50bc69c999e812211903b169799208f2a8", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "aa978504bcf76511efdc22d580ba08e2279caab1066b76bb9aa81c4a1e0a32a5"}, @@ -54,7 +55,8 @@ "guardian": {:hex, :guardian, "2.4.0", "efbbb397ecca881bb548560169922fc4433a05bc98c2eb96a7ed88ede9e17d64", [:mix], [{:jose, "~> 1.11.9", [hex: :jose, repo: "hexpm", optional: false]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "5c80103a9c538fbc2505bf08421a82e8f815deba9eaedb6e734c66443154c518"}, "guardian_db": {:hex, :guardian_db, "2.1.0", "ec95a9d99cdd1e550555d09a7bb4a340d8887aad0697f594590c2fd74be02426", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:guardian, "~> 1.0 or ~> 2.0", [hex: :guardian, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.13", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "f8e7d543ac92c395f3a7fd5acbe6829faeade57d688f7562e2f0fca8f94a0d70"}, "hackney": {:hex, :hackney, "1.25.0", "390e9b83f31e5b325b9f43b76e1a785cbdb69b5b6cd4e079aa67835ded046867", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "7209bfd75fd1f42467211ff8f59ea74d6f2a9e81cbcee95a56711ee79fd6b1d4"}, - "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, + "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, + "httpoison": {:hex, :httpoison, "2.2.3", "a599d4b34004cc60678999445da53b5e653630651d4da3d14675fedc9dd34bd6", [:mix], [{:hackney, "~> 1.21", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "fa0f2e3646d3762fdc73edb532104c8619c7636a6997d20af4003da6cfc53e53"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "inch_ex": {:hex, :inch_ex, "2.1.0-rc.1", "7642a8902c0d2ed5d9b5754b2fc88fedf630500d630fc03db7caca2e92dedb36", [:mix], [{:bunt, "~> 0.2", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "4ceee988760f9382d1c1d0b93ea5875727f6071693e89a0a3c49c456ef1be75d"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, @@ -67,13 +69,15 @@ "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mimerl": {:hex, :mimerl, "1.4.0", "3882a5ca67fbbe7117ba8947f27643557adec38fa2307490c4c4207624cb213b", [:rebar3], [], "hexpm", "13af15f9f68c65884ecca3a3891d50a7b57d82152792f3e19d88650aa126b144"}, + "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, "mock": {:hex, :mock, "0.3.9", "10e44ad1f5962480c5c9b9fa779c6c63de9bd31997c8e04a853ec990a9d841af", [:mix], [{:meck, "~> 0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "9e1b244c4ca2551bb17bb8415eed89e40ee1308e0fbaed0a4fdfe3ec8a4adbd3"}, "mox": {:hex, :mox, "1.2.0", "a2cd96b4b80a3883e3100a221e8adc1b98e4c3a332a8fc434c39526babafd5b3", [:mix], [{:nimble_ownership, "~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}], "hexpm", "c7b92b3cc69ee24a7eeeaf944cd7be22013c52fcb580c1f33f50845ec821089a"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_ownership": {:hex, :nimble_ownership, "1.0.1", "f69fae0cdd451b1614364013544e66e4f5d25f36a2056a9698b793305c5aa3a6", [:mix], [], "hexpm", "3825e461025464f519f3f3e4a1f9b68c47dc151369611629ad08b636b73bb22d"}, + "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "oban": {:hex, :oban, "2.18.0", "092d20bfd3d70c7ecb70960f8548d300b54bb9937c7f2e56b388f3a9ed02ec68", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "aace1eff6f8227ae38d4274af967d96f051c2f0a5152f2ef9809dd1f97866745"}, - "openai": {:hex, :openai, "0.4.1", "c141283d35fed72eaeee22397f0a3eaa93515283b3a698cd3aa0af9301bb5fc5", [:mix], [{:httpoison, "~> 1.8", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "d958993926474362715152d5d4f68444038d53e649641fe1af2fcb26ca8bbe90"}, - "openid_connect": {:hex, :openid_connect, "0.2.2", "c05055363330deab39ffd89e609db6b37752f255a93802006d83b45596189c0b", [:mix], [{:httpoison, "~> 1.2", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "735769b6d592124b58edd0582554ce638524c0214cd783d8903d33357d74cc13"}, + "openai": {:hex, :openai, "0.6.2", "48ee0dc74f4d0327ebf78eaeeed8c3595e10eca88fd2a7cdd8b53473645078d6", [:mix], [{:httpoison, "~> 2.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "508c8c2937ef8627d111d9142ff9cc284d39cd0c9b8244339551ac5f5fe0e643"}, + "openid_connect": {:hex, :openid_connect, "1.0.0", "afd586d9f011cf83035dc18cc1e6cc0994b7fdf04ece91f91ba8a3f0ca5e343b", [:mix], [{:finch, "~> 0.14", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "1abc2009fad37321996b64ac4b36d8113b309b697707c0774ed6d3dbad9a5005"}, "parallel_stream": {:hex, :parallel_stream, "1.1.0", "f52f73eb344bc22de335992377413138405796e0d0ad99d995d9977ac29f1ca9", [:mix], [], "hexpm", "684fd19191aedfaf387bbabbeb8ff3c752f0220c8112eb907d797f4592d6e871"}, "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, "phoenix": {:hex, :phoenix, "1.8.1", "865473a60a979551a4879db79fbfb4503e41cd809e77c85af79716578b6a456d", [:mix], [{:bandit, "~> 1.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "84d77d2b2e77c3c7e7527099bd01ef5c8560cd149c036d6b3a40745f11cd2fb2"}, From e95766e5a7f609af5fee04b903746bb0b6be53db Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Thu, 30 Oct 2025 23:25:27 +0800 Subject: [PATCH 68/77] rm un-used --- lib/cadet_web/controllers/generate_ai_comments.ex | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 5ccefcbf5..f8e1d8be0 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -185,13 +185,6 @@ defmodule CadetWeb.AICodeAnalysisController do defp format_autograding_results(results), do: inspect(results) - def call_llm_endpoint(llm_api_url, input, headers) do - HTTPoison.post(llm_api_url, input, headers, - timeout: 60_000, - recv_timeout: 60_000 - ) - end - defp analyze_code( conn, %{ From 1091e7764c7f81f80e433a8f040d24762b53a949 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Fri, 31 Oct 2025 01:05:32 +0800 Subject: [PATCH 69/77] Merge all prompts to :prompts to preserve abstraction --- lib/cadet/assessments/submission.ex | 1 - .../admin_views/admin_grading_view.ex | 32 +- .../controllers/generate_ai_comments.ex | 86 ++- lib/cadet_web/helpers/assessments_helpers.ex | 1 - test.txt | 619 ++++++++++++++++++ .../admin_grading_controller_test.exs | 24 +- .../assessments_controller_test.exs | 1 - 7 files changed, 720 insertions(+), 44 deletions(-) create mode 100644 test.txt diff --git a/lib/cadet/assessments/submission.ex b/lib/cadet/assessments/submission.ex index 5ef3b0574..658f79f1f 100644 --- a/lib/cadet/assessments/submission.ex +++ b/lib/cadet/assessments/submission.ex @@ -25,7 +25,6 @@ defmodule Cadet.Assessments.Submission do belongs_to(:team, Team) belongs_to(:unsubmitted_by, CourseRegistration) has_many(:answers, Answer, on_delete: :delete_all) - has_many(:ai_comments, Cadet.AIComments.AIComment, on_delete: :delete_all) timestamps() end diff --git a/lib/cadet_web/admin_views/admin_grading_view.ex b/lib/cadet_web/admin_views/admin_grading_view.ex index fc4be7ba3..f6a90d17d 100644 --- a/lib/cadet_web/admin_views/admin_grading_view.ex +++ b/lib/cadet_web/admin_views/admin_grading_view.ex @@ -2,12 +2,18 @@ defmodule CadetWeb.AdminGradingView do use CadetWeb, :view import CadetWeb.AssessmentsHelpers + alias CadetWeb.AICodeAnalysisController def render("show.json", %{course: course, answers: answers, assessment: assessment}) do %{ assessment: render_one(assessment, CadetWeb.AdminGradingView, "assessment.json", as: :assessment), - answers: render_many(answers, CadetWeb.AdminGradingView, "grading_info.json", as: :answer), + answers: + render_many(answers, CadetWeb.AdminGradingView, "grading_info.json", + as: :answer, + course: course, + assessment: assessment + ), enable_llm_grading: course.enable_llm_grading } end @@ -21,8 +27,7 @@ defmodule CadetWeb.AdminGradingView do coverPicture: assessment.cover_picture, number: assessment.number, story: assessment.story, - reading: assessment.reading, - llm_assessment_prompt: assessment.llm_assessment_prompt + reading: assessment.reading } end @@ -149,13 +154,14 @@ defmodule CadetWeb.AdminGradingView do } end - def render("grading_info.json", %{answer: answer}) do + def render("grading_info.json", %{answer: answer, course: course, assessment: assessment}) do transform_map_for_view(answer, %{ id: & &1.id, + prompts: &build_prompts(&1, course, assessment), ai_comments: &extract_ai_comments_per_answer(&1.id, &1.ai_comments), student: &extract_student_data(&1.submission.student), team: &extract_team_data(&1.submission.team), - question: &build_grading_question/1, + question: &build_grading_question(&1, course, assessment), solution: &(&1.question.question["solution"] || ""), grade: &build_grade/1 }) @@ -206,14 +212,26 @@ defmodule CadetWeb.AdminGradingView do end end - defp build_grading_question(answer) do - %{question: answer.question} + defp build_grading_question(answer, course, assessment) do + %{question: answer.question |> Map.delete(:llm_prompt)} |> build_question_by_question_config(true) |> Map.put(:answer, answer.answer["code"] || answer.answer["choice_id"]) |> Map.put(:autogradingStatus, answer.autograding_status) |> Map.put(:autogradingResults, answer.autograding_results) end + defp build_prompts(answer, course, assessment) do + if course.enable_llm_grading do + AICodeAnalysisController.create_final_messages( + course.llm_course_level_prompt, + assessment.llm_assessment_prompt, + answer + ) + else + [] + end + end + defp build_grade(answer = %{grader: grader}) do transform_map_for_view(answer, %{ grader: grader_builder(grader), diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index f8e1d8be0..8ee8ccab5 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -5,8 +5,7 @@ defmodule CadetWeb.AICodeAnalysisController do require Logger alias Cadet.{Assessments, AIComments, Courses} - alias CadetWeb.AICodeAnalysisController - alias CadetWeb.AICommentsHelpers + alias CadetWeb.{AICodeAnalysisController, AICommentsHelpers} # For logging outputs to both database and file defp save_comment(answer_id, raw_prompt, answers_json, response, error \\ nil) do @@ -150,8 +149,9 @@ defmodule CadetWeb.AICodeAnalysisController do end defp format_system_prompt(course_prompt, assessment_prompt, answer) do - (course_prompt || "") <> - "\n\n" <> + "**Course Level Prompt:**\n\n" <> + (course_prompt || "") <> + "\n\n**Assessment Level Prompt:**" <> (assessment_prompt || "") <> "\n\n" <> """ @@ -175,6 +175,22 @@ defmodule CadetWeb.AICodeAnalysisController do """ end + def create_final_messages( + course_prompt, + assessment_prompt, + answer + ) do + formatted_answer = + answer + |> format_student_answer() + |> Jason.encode!() + + [ + %{role: "system", content: format_system_prompt(course_prompt, assessment_prompt, answer)}, + %{role: "user", content: formatted_answer} + ] + end + defp format_autograding_results(nil), do: "N/A" defp format_autograding_results(results) when is_list(results) do @@ -196,50 +212,68 @@ defmodule CadetWeb.AICodeAnalysisController do assessment_prompt: assessment_prompt } ) do - formatted_answer = - answer - |> format_student_answer() - |> Jason.encode!() - - system_prompt = format_system_prompt(course_prompt, assessment_prompt, answer) # Combine prompts if llm_prompt exists + final_messages = + create_final_messages( + course_prompt, + assessment_prompt, + answer + ) + input = [ model: llm_model, - messages: [ - %{role: "system", content: system_prompt}, - %{role: "user", content: formatted_answer} - ] + messages: final_messages ] case OpenAI.chat_completion(input, %OpenAI.Config{ api_url: llm_api_url, api_key: api_key, - http_options: [ - timeout: 60_000, # connect timeout - recv_timeout: 60_000 # response timeout - ] + http_options: [ + # connect timeout + timeout: 60_000, + # response timeout + recv_timeout: 60_000 + ] }) do {:ok, %{choices: [%{"message" => %{"content" => content}} | _]}} -> - save_comment(answer.id, system_prompt, formatted_answer, content) + save_comment( + answer.id, + Enum.at(final_messages, 0).content, + Enum.at(final_messages, 1).content, + content + ) + comments_list = String.split(content, "|||") - filtered_comments = - Enum.filter(comments_list, fn comment -> - String.trim(comment) != "" - end) + filtered_comments = + Enum.filter(comments_list, fn comment -> + String.trim(comment) != "" + end) - json(conn, %{"comments" => filtered_comments}) + json(conn, %{"comments" => filtered_comments}) {:ok, other} -> - save_comment(answer.id, system_prompt, formatted_answer, other, "Unexpected JSON shape") + save_comment( + answer.id, + Enum.at(final_messages, 0).content, + Enum.at(final_messages, 1).content, + other, + "Unexpected JSON shape" + ) conn |> put_status(:bad_gateway) |> text("Unexpected response format from LLM") {:error, reason} -> - save_comment(answer.id, system_prompt, formatted_answer, nil, inspect(reason)) + save_comment( + answer.id, + Enum.at(final_messages, 0).content, + Enum.at(final_messages, 1).content, + nil, + inspect(reason) + ) conn |> put_status(:internal_server_error) diff --git a/lib/cadet_web/helpers/assessments_helpers.ex b/lib/cadet_web/helpers/assessments_helpers.ex index 9af0bd36e..8f2dec32f 100644 --- a/lib/cadet_web/helpers/assessments_helpers.ex +++ b/lib/cadet_web/helpers/assessments_helpers.ex @@ -181,7 +181,6 @@ defmodule CadetWeb.AssessmentsHelpers do prepend: "prepend", solutionTemplate: "template", postpend: "postpend", - llm_prompt: "llm_prompt", testcases: build_testcases(all_testcases?) }) diff --git a/test.txt b/test.txt new file mode 100644 index 000000000..779ccd3d1 --- /dev/null +++ b/test.txt @@ -0,0 +1,619 @@ +Compiling 3 files (.ex) +Compiling 3 files (.ex) +Generated cadet app +cadet: generated priv/static/swagger.json + +01:03:16.715 [info] Migrations already up +Running ExUnit with seed: 184121, max_cases: 44 + +.............................[ASSESSMENT_SUBMISSION] SENDING_OUT +.....[AVENGER_BACKLOG] course-level disabled +[AVENGER_BACKLOG] SENDING_OUT +[AVENGER_BACKLOG] SENDING_OUT +. + + 1) test Manage assets delete ok file (Cadet.Assets.AssetsTest) + test/cadet/assets/assets_test.exs:31 + ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/model_delete_asset#2.json. + Delete the current cassette with [mix vcr.delete] and re-record. + + Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/1/testFolder/test.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=eb4636f36507f3438e88be438b49ac665c3dd7ee8469dcc7432853f4eb9c7ba6"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170319Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] + + code: assert Assets.delete_object(prefix(1), "testFolder", "test.png") === :ok + stacktrace: + (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 + (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 + (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 + (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 + (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 + (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 + (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 + (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 + (cadet 0.0.1) lib/cadet/assets/assets.ex:55: Cadet.Assets.Assets.delete_object/3 + test/cadet/assets/assets_test.exs:33: (test) + + + + 2) test Manage assets access another course with 0 folder (Cadet.Assets.AssetsTest) + test/cadet/assets/assets_test.exs:18 + ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/model_list_assets#2.json. + Delete the current cassette with [mix vcr.delete] and re-record. + + Request: [:get, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/?prefix=courses-dev%2F2%2FtestFolder%2F", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=3e2bfe468373c6b076b8795389ea54372078104e8433755851457ca9ae85d380"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170319Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] + + code: assert Assets.list_assets(prefix(2), "testFolder") === [] + stacktrace: + (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 + (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 + (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 + (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 + (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 + (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 + (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 + (ex_aws 2.5.9) lib/ex_aws.ex:85: ExAws.request!/2 + (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:7: anonymous fn/4 in ExAws.S3.Lazy.stream_objects!/3 + (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:18: anonymous fn/2 in ExAws.S3.Lazy.stream_objects!/3 + (elixir 1.18.3) lib/stream.ex:1571: Stream.do_resource/5 + (elixir 1.18.3) lib/enum.ex:4515: Enum.map/2 + test/cadet/assets/assets_test.exs:20: (test) + + + + 3) test Manage assets upload existing file (Cadet.Assets.AssetsTest) + test/cadet/assets/assets_test.exs:37 + ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/model_upload_asset#1.json. + Delete the current cassette with [mix vcr.delete] and re-record. + + Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/1/testFolder/test2.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=13c15ffce68a98d964248228c70eede720fe0d78fb99662f0fd89b6a187e7fb6"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170320Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] + + code: assert Assets.upload_to_s3( + stacktrace: + (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 + (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 + (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 + (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 + (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 + (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 + (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 + (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 + (cadet 0.0.1) lib/cadet/assets/assets.ex:18: Cadet.Assets.Assets.upload_to_s3/4 + test/cadet/assets/assets_test.exs:39: (test) + + + + 4) test Manage assets delete nonexistent file (Cadet.Assets.AssetsTest) + test/cadet/assets/assets_test.exs:24 + ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/model_delete_asset#1.json. + Delete the current cassette with [mix vcr.delete] and re-record. + + Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/1/testFolder/test4.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=12083bf68aeedc6ceb05f9ab4e5c7bf69034aef06b99600594755aed667473aa"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170320Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] + + code: assert Assets.delete_object(prefix(1), "testFolder", "test4.png") === + stacktrace: + (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 + (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 + (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 + (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 + (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 + (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 + (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 + (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 + (cadet 0.0.1) lib/cadet/assets/assets.ex:55: Cadet.Assets.Assets.delete_object/3 + test/cadet/assets/assets_test.exs:26: (test) + + + + 5) test Manage assets accessible folder (Cadet.Assets.AssetsTest) + test/cadet/assets/assets_test.exs:8 + ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/model_list_assets#1.json. + Delete the current cassette with [mix vcr.delete] and re-record. + + Request: [:get, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/?prefix=courses-dev%2F1%2FtestFolder%2F", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=3fa280761ada3502c164c277e7aaa3781a83db339553d456ec73e18732184bd4"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170320Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] + + code: assert Assets.list_assets(prefix(1), "testFolder") === [ + stacktrace: + (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 + (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 + (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 + (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 + (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 + (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 + (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 + (ex_aws 2.5.9) lib/ex_aws.ex:85: ExAws.request!/2 + (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:7: anonymous fn/4 in ExAws.S3.Lazy.stream_objects!/3 + (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:18: anonymous fn/2 in ExAws.S3.Lazy.stream_objects!/3 + (elixir 1.18.3) lib/stream.ex:1571: Stream.do_resource/5 + (elixir 1.18.3) lib/enum.ex:4515: Enum.map/2 + test/cadet/assets/assets_test.exs:10: (test) + + + + 6) test Manage assets upload ok file (Cadet.Assets.AssetsTest) + test/cadet/assets/assets_test.exs:49 + ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/model_upload_asset#2.json. + Delete the current cassette with [mix vcr.delete] and re-record. + + Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/1/testFolder/test1.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=5364a5f81c7020737d88523c43daa502d6337034f8953925d11c3d3d2ac0439d"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170320Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] + + code: assert Assets.upload_to_s3( + stacktrace: + (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 + (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 + (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 + (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 + (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 + (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 + (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 + (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 + (cadet 0.0.1) lib/cadet/assets/assets.ex:18: Cadet.Assets.Assets.upload_to_s3/4 + test/cadet/assets/assets_test.exs:51: (test) + +.................................................01:03:21.444 request_id=GHNTmk5EyRz9IDkAADfC [error] Failed to create course for user 7361: viewable is invalid +01:03:21.444 request_id=GHNTmk5EyRz9IDkAADfC [error] Invalid parameters provided by user 7361 while creating a course. +..01:03:21.456 request_id=GHNTmk96Kd29nP4AADgC [error] User 7363 has exceeded the limit of 5 admin courses. +.01:03:21.459 request_id=GHNTmk-pD6MfhdcAABMB [error] Failed to create course for user 7364: source_chapter source chapter and source variant must be present together; can't be blank +.01:03:21.459 request_id=GHNTmk-pD6MfhdcAABMB [error] Invalid parameters provided by user 7364 while creating a course. +01:03:21.462 request_id=GHNTmk_Wkn1ssGsAABNB [error] User record not found for user 7365 in course 29701 +................................................01:03:22.199 request_id=GHNTmnvI_oX9Yp8AADsC [error] User record not found for user 7412 in course 29869 +.01:03:22.202 request_id=GHNTmnvs2Bg53agAAAtH [error] Failed to update course configuration for course 29869: source_variant is invalid +..01:03:22.206 request_id=GHNTmnwy0jC9wigAAAvH [error] User record not found for user 7415 in course 29872 +.................01:03:22.261 request_id=GHNTmn94bNTiTjEAAA6H [error] Failed to update course configuration for course 29888: source_chapter source chapter and source variant must be present together +..01:03:22.264 request_id=GHNTmn-qj43xDksAAA9E [error] User record not found for user 7433 in course 29890 +..........01:03:22.290 request_id=GHNTmoEzrqBLWjYAAD1C [error] Team creation failed for assessment 15139 - duplicate students +.....01:03:22.323 request_id=GHNTmoMo0TImw0QAABFE [error] User record not found for user 7453 in course 29925 +.01:03:22.331 request_id=GHNTmoOdXfk5HtMAABGE [error] Cannot delete team 207 - team has submitted answers +..01:03:22.345 request_id=GHNTmoSBEdG2BRoAAD9C [error] User record not found for user 7461 in course 29935 +.01:03:22.351 request_id=GHNTmoTZKrxMEfQAAD-C [error] Team creation failed for assessment 15147 - students already assigned to teams +......01:03:22.364 request_id=GHNTmoWV5G0FRCAAABIE [error] Team creation failed for assessment 15148 - teams exceed maximum size +..01:03:22.369 request_id=GHNTmoXrndIDf3YAAEDC [error] User record not found for user 7472 in course 29956 +.01:03:22.374 request_id=GHNTmoYvCJyQI1UAABJE [error] Team creation failed for assessment 15150 - students not enrolled in course +....01:03:22.377 request_id=GHNTmoZsN0WH2fMAAEHC [error] Failed to update research agreement for user 7477. Status: bad_request, Message: agreed_to_research is invalid. +.01:03:22.380 request_id=GHNTmoaNGkbGZX4AABKE [error] Failed to update game states for user 7478. Status: bad_request, Message: game_states is invalid. +.........01:03:22.465 request_id=GHNTmouj56eiGZsAABQE [error] User 7489 is not enrolled in course 29997 +.01:03:22.465 request_id=GHNTmouj56eiGZsAABQE [error] Failed to update latest viewed course for user 7489. Status: bad_request, Message: user is not in the course. +..........01:03:22.478 request_id=GHNTmoxq8LKyAZsAABRE [error] Failed to upload sourcecast for user 7492: title can't be blank +audio can't be blank +playbackData can't be blank +.............01:03:22.549 request_id=GHNTmpCh1KlKr24AABZE [error] Sourcecast 1 not found +...01:03:22.556 request_id=GHNTmpEOrZzDVVsAAEeC [error] User record not found for user 7524 in course 30042 +..........01:03:23.067 request_id=GHNTmq-OQ5ctFOsAABQB [error] User record not found for user 7542 in course 30180 +.......01:03:24.174 request_id=GHNTmvF4pV42OOcAABVB [error] Cannot delete story 78 - user not allowed to manage stories from another course +....01:03:24.191 request_id=GHNTmvJ6Qgl2BK0AAEoC [error] Cannot update story 80 - user not allowed to manage stories from another course +........01:03:24.212 request_id=GHNTmvO-cZH_0WMAAEuC [error] Failed to create story for course 30463: title can't be blank +close_at can't be blank +open_at can't be blank +filenames can't be blank +. + + 7) test GET /v2/courses/:course_id/admin/generate-comments/:submissionid/:questionid errors out when given an invalid submission (CadetWeb.AICodeAnalysisControllerTest) + test/cadet_web/controllers/ai_code_analysis_controller_test.exs:101 + ** (UndefinedFunctionError) function Cadet.Courses.Course.encrypt_llm_api_key/1 is undefined or private + stacktrace: + (cadet 0.0.1) Cadet.Courses.Course.encrypt_llm_api_key("test_key") + test/cadet_web/controllers/ai_code_analysis_controller_test.exs:12: CadetWeb.AICodeAnalysisControllerTest.__ex_unit_setup_1/1 + test/cadet_web/controllers/ai_code_analysis_controller_test.exs:3: CadetWeb.AICodeAnalysisControllerTest.__ex_unit__/2 + + + + 8) test GET /v2/courses/:course_id/admin/generate-comments/:submissionid/:questionid LLM endpoint returns an invalid response - should log errors in database (CadetWeb.AICodeAnalysisControllerTest) + test/cadet_web/controllers/ai_code_analysis_controller_test.exs:135 + ** (UndefinedFunctionError) function Cadet.Courses.Course.encrypt_llm_api_key/1 is undefined or private + stacktrace: + (cadet 0.0.1) Cadet.Courses.Course.encrypt_llm_api_key("test_key") + test/cadet_web/controllers/ai_code_analysis_controller_test.exs:12: CadetWeb.AICodeAnalysisControllerTest.__ex_unit_setup_1/1 + test/cadet_web/controllers/ai_code_analysis_controller_test.exs:3: CadetWeb.AICodeAnalysisControllerTest.__ex_unit__/2 + + + + 9) test GET /v2/courses/:course_id/admin/generate-comments/:submissionid/:questionid success with happy path, admin and staff (CadetWeb.AICodeAnalysisControllerTest) + test/cadet_web/controllers/ai_code_analysis_controller_test.exs:38 + ** (UndefinedFunctionError) function Cadet.Courses.Course.encrypt_llm_api_key/1 is undefined or private + stacktrace: + (cadet 0.0.1) Cadet.Courses.Course.encrypt_llm_api_key("test_key") + test/cadet_web/controllers/ai_code_analysis_controller_test.exs:12: CadetWeb.AICodeAnalysisControllerTest.__ex_unit_setup_1/1 + test/cadet_web/controllers/ai_code_analysis_controller_test.exs:3: CadetWeb.AICodeAnalysisControllerTest.__ex_unit__/2 + +......01:03:24.243 [error] Failed to upsert achievement eb61dfa8-51c3-4b5d-89f8-3239de478c5d: course_id can't be blank +...................................................................................................................................................................???????01:03:51.816 [error] Cannot update course 32669 - course not found +01:03:51.817 [error] Failed to update course configuration for course 32669: source_chapter is invalid +01:03:51.818 [error] Failed to update course configuration for course 32669: source_variant is invalid + + + 9) Cadet.Auth.Providers.OpenIDTest: failure on setup_all callback, all tests have been invalidated + ** (UndefinedFunctionError) function OpenIDConnect.Worker.start_link/1 is undefined (module OpenIDConnect.Worker is not available) + stacktrace: + OpenIDConnect.Worker.start_link([test: [discovery_document_uri: "http://localhost:43705/.well-known/openid-configuration", client_id: "dummy", client_secret: "dummydummy", response_type: "code", scope: "openid profile"]]) + test/cadet/auth/providers/openid/openid_test.exs:67: Cadet.Auth.Providers.OpenIDTest.__ex_unit_setup_all_0/1 + test/cadet/auth/providers/openid/openid_test.exs:1: Cadet.Auth.Providers.OpenIDTest.__ex_unit__/2 + +......................01:03:51.923 [error] Course 32701 not found +.......01:03:51.968 request_id=GHNToWn8V_7Cl2MAACsB [error] Failed to answer question 16779 - invalid parameters +01:03:51.971 request_id=GHNToWo3LEPCl2MAACvB [error] Failed to answer question 16778 - invalid parameters +01:03:51.978 request_id=GHNToWqXAmfCl2MAACwB [error] Invalid vote for question 16780 by user 8189 +......01:03:52.134 request_id=GHNToXPjPpRZHc4AADFB [error] Failed to answer question 16798 - invalid parameters +01:03:52.137 request_id=GHNToXQfkqhZHc4AADGB [error] Failed to answer question 16797 - invalid parameters +01:03:52.143 request_id=GHNToXR2X6VZHc4AADHB [error] Invalid vote for question 16799 by user 8199 +.....01:03:52.243 request_id=GHNToXpb0hjp8p8AADTB [error] Failed to answer question 16813 - invalid parameters +01:03:52.248 request_id=GHNToXqg0eXp8p8AADUB [error] Failed to answer question 16812 - invalid parameters +01:03:52.255 request_id=GHNToXsoZLHp8p8AADVB [error] Invalid vote for question 16814 by user 8207 +..........01:03:52.445 request_id=GHNToYZ1N8IJtM4AADqB [error] Failed to answer question 16845 - submission already finalized +.........01:03:52.635 request_id=GHNToZHdDoc49jkAABbE [error] Login failed for provider 'test'. +.01:03:52.668 request_id=GHNToZPlm7-ESS8AACZD [error] SAML redirect failed for provider 'saml'. +01:03:52.684 request_id=GHNToZThD4VlPLwAADvB [error] Logout request failed due to missing parameters. +01:03:52.685 request_id=GHNToZTqZN5ByxwAABcE [error] Login failed for provider 'test'. +01:03:52.685 request_id=GHNToZTz5gcz2isAABdE [error] Invalid refresh token provided. + + + 10) test GET /auth/saml_redirect success (CadetWeb.AuthControllerTest) + test/cadet_web/controllers/auth_controller_test.exs:104 + ** (RuntimeError) expected response with status 302, got: 500, with body: + "Unknown error: Invalid or nonexistent provider config" + code: assert response(conn, 302) + stacktrace: + (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:373: Phoenix.ConnTest.response/2 + test/cadet_web/controllers/auth_controller_test.exs:110: (test) + +...01:03:52.713 request_id=GHNToZaS1TPB38cAABTF [error] SAML redirect failed for provider 'saml'. + + +01:03:52.726 request_id=GHNToZdZV8GHq9IAACaD [error] Refresh request failed due to missing parameters. + 11) test GET /auth/saml_redirect missing username attribute (CadetWeb.AuthControllerTest) + test/cadet_web/controllers/auth_controller_test.exs:136 + ** (RuntimeError) expected response with status 400, got: 500, with body: + "Unknown error: Invalid or nonexistent provider config" + code: assert response(conn, 400) == "Unable to validate token: Missing username attribute!" + stacktrace: + (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:373: Phoenix.ConnTest.response/2 + test/cadet_web/controllers/auth_controller_test.exs:140: (test) + +...01:03:52.751 request_id=GHNToZjTB4u9BmkAADxB [error] Invalid refresh token provided. +.01:03:52.774 request_id=GHNToZo2S_IetKQAADVG [error] SAML redirect failed for provider 'saml'. + + + 12) test GET /auth/saml_redirect missing name attribute (CadetWeb.AuthControllerTest) + test/cadet_web/controllers/auth_controller_test.exs:129 + ** (RuntimeError) expected response with status 400, got: 500, with body: + "Unknown error: Invalid or nonexistent provider config" + code: assert response(conn, 400) == "Unable to validate token: Missing name attribute!" + stacktrace: + (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:373: Phoenix.ConnTest.response/2 + test/cadet_web/controllers/auth_controller_test.exs:133: (test) + +.01:03:52.804 request_id=GHNToZwFd-HOmCoAADyB [error] Login failed for provider 'test'. +.01:03:52.813 request_id=GHNToZySFFocBfMAADzB [error] Login request failed due to missing parameters. +.01:03:52.814 request_id=GHNToZycOvc6H20AADWG [error] Invalid token provided for logout. +.01:03:52.837 request_id=GHNToZ34blUgJNsAABVF [error] SAML redirect request failed due to missing parameters. +..01:03:52.851 request_id=GHNToZ7NKHt-8qoAABWF [error] Invalid refresh token provided. +.01:03:52.872 request_id=GHNToaAOFW-TR0wAABXF [error] Login failed for provider 'test'. +.01:03:52.912 request_id=GHNToaI1e9IXrqAAACbD [error] SAML redirect failed for provider 'saml'. + + + 13) test GET /auth/saml_redirect missing SAML assertion (CadetWeb.AuthControllerTest) + test/cadet_web/controllers/auth_controller_test.exs:122 + ** (RuntimeError) expected response with status 400, got: 500, with body: + "Unknown error: Invalid or nonexistent provider config" + code: assert response(conn, 400) == "Unable to validate token: Missing SAML assertion!" + stacktrace: + (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:373: Phoenix.ConnTest.response/2 + test/cadet_web/controllers/auth_controller_test.exs:126: (test) + +.................................01:03:53.165 [error] Failed to enroll user 8202 in course 33061: role is invalid +.................................01:03:53.451 [error] No user found in conn.assigns.current_user, please check the AssignCurrentUser plug +..01:03:53.453 [error] Rate limit of 500 exceeded for user 1 +.........01:03:53.586 request_id=GHNTocqFn5HTOpIAABoE [error] Device registration 101 not found for user 8321 +.01:03:53.592 request_id=GHNTocrtQ5n2GAwAACjD [error] Device registration 103 not found for user 8322 +....01:03:53.775 request_id=GHNTodWkWcOXgEcAABpE [error] Device registration 105 not found for user 8324 +..........01:03:53.874 request_id=GHNTodvC06qXzaQAABrE [error] Device registration 107 not found for user 8327 +....01:03:53.974 request_id=GHNToeG2rp80nvkAABeF [error] Failed to rename device registration 108: title can't be blank +...............01:03:54.085 request_id=GHNToegYNL-spbgAAEBB [error] User 8364 is not authorized to access conversation 9. +.01:03:54.085 request_id=GHNToegYNL-spbgAAEBB [error] Conversation not found for user 8364, conversation 9. +01:03:54.089 request_id=GHNToeiMruNxmcQAABsE [error] Conversation -1 not found for user 8366. +.01:03:54.089 request_id=GHNToeiMruNxmcQAABsE [error] Conversation not found for user 8366, conversation -1. +01:03:54.246 request_id=GHNTofHfN5t0i_EAAECB [error] User 8367 is not authorized to access conversation 10. +01:03:54.246 request_id=GHNTofHfN5t0i_EAAECB [error] Conversation not found for user 8367, conversation 10. +...01:03:54.439 request_id=GHNTof172nihNroAABgF [error] Section is missing for user 8370. +..01:03:54.444 request_id=GHNTof3CqxMzFHAAAEDB [error] Message too long for user 8371. Length: 1001. +.............................................01:04:08.228 [error] Cannot update story 86 - story not found +..01:04:08.231 [error] Cannot delete story 88 - user not allowed to manage stories from another course +.01:04:08.233 [error] Cannot delete story 90 - story not found +....01:04:08.239 [error] Cannot update story 96 - user not allowed to manage stories from another course +....... + + 14) test course with custom assets_prefix upload file (CadetWeb.AdminAssetsControllerTest) + test/cadet_web/admin_controllers/admin_assets_controller_test.exs:262 + ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_upload_asset#3.json. + Delete the current cassette with [mix vcr.delete] and re-record. + + Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/this-is-a-prefix/testFolder/nestedFolder/test.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=3b74e4c99c27399a00df997de8e7f7cb7b51e3000eea2b45d189e7300fb0414a"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170408Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] + + code: post(conn, build_url(course_id, "testFolder/nestedFolder/test.png"), %{ + stacktrace: + (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 + (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 + (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 + (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 + (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 + (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 + (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 + (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 + (cadet 0.0.1) lib/cadet/assets/assets.ex:18: Cadet.Assets.Assets.upload_to_s3/4 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:42: CadetWeb.AdminAssetsController.upload/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 + (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 + (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:225: Phoenix.ConnTest.dispatch/5 + test/cadet_web/admin_controllers/admin_assets_controller_test.exs:269: (test) + +. + + 15) test ok request index file (CadetWeb.AdminAssetsControllerTest) + test/cadet_web/admin_controllers/admin_assets_controller_test.exs:132 + ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_list_assets#1.json. + Delete the current cassette with [mix vcr.delete] and re-record. + + Request: [:get, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/?prefix=courses-dev%2F117%2FtestFolder%2F", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=83cd5736d311d1ece755eb34f0a86ef7d6f8a5d105191173d8a9cbb293794b62"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170408Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] + + stacktrace: + (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 + (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 + (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 + (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 + (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 + (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 + (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 + (ex_aws 2.5.9) lib/ex_aws.ex:85: ExAws.request!/2 + (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:7: anonymous fn/4 in ExAws.S3.Lazy.stream_objects!/3 + (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:18: anonymous fn/2 in ExAws.S3.Lazy.stream_objects!/3 + (elixir 1.18.3) lib/stream.ex:1571: Stream.do_resource/5 + (elixir 1.18.3) lib/enum.ex:4515: Enum.map/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:12: CadetWeb.AdminAssetsController.index/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 + (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 + (hackney 1.25.0) :hackney.request(:get, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/?prefix=courses-dev%2F117%2FtestFolder%2F", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=83cd5736d311d1ece755eb34f0a86ef7d6f8a5d105191173d8a9cbb293794b62"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170408Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]) + + + + 16) test course with custom assets_prefix delete file (CadetWeb.AdminAssetsControllerTest) + test/cadet_web/admin_controllers/admin_assets_controller_test.exs:249 + ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_delete_asset#3.json. + Delete the current cassette with [mix vcr.delete] and re-record. + + Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/this-is-a-prefix/testFolder/nestedFolder/test2.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=3ad6957b6c3a2a4891a6dc436a9e46b8355f4c0173f097ddf647bac419ef8eb0"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170408Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] + + code: conn = delete(conn, build_url(course_id, "testFolder/nestedFolder/test2.png")) + stacktrace: + (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 + (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 + (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 + (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 + (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 + (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 + (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 + (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 + (cadet 0.0.1) lib/cadet/assets/assets.ex:55: Cadet.Assets.Assets.delete_object/3 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:23: CadetWeb.AdminAssetsController.delete/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 + (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 + (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:225: Phoenix.ConnTest.dispatch/5 + test/cadet_web/admin_controllers/admin_assets_controller_test.exs:255: (test) + +.. + + 17) test nested filename request delete file (CadetWeb.AdminAssetsControllerTest) + test/cadet_web/admin_controllers/admin_assets_controller_test.exs:207 + ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_delete_asset#2.json. + Delete the current cassette with [mix vcr.delete] and re-record. + + Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/117/testFolder/nestedFolder/test2.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=f816c2b657d12b86c0030828709c8f725398e9af7eb25bf3b797b24ee7331950"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170409Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] + + code: conn = delete(conn, build_url(course_id, "testFolder/nestedFolder/test2.png")) + stacktrace: + (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 + (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 + (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 + (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 + (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 + (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 + (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 + (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 + (cadet 0.0.1) lib/cadet/assets/assets.ex:55: Cadet.Assets.Assets.delete_object/3 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:23: CadetWeb.AdminAssetsController.delete/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 + (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 + (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:225: Phoenix.ConnTest.dispatch/5 + test/cadet_web/admin_controllers/admin_assets_controller_test.exs:211: (test) + + + + 18) test nested filename request upload file (CadetWeb.AdminAssetsControllerTest) + test/cadet_web/admin_controllers/admin_assets_controller_test.exs:218 + ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_upload_asset#2.json. + Delete the current cassette with [mix vcr.delete] and re-record. + + Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/117/testFolder/nestedFolder/test.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=b276e93ad753e86da3b5d8b0b43472e3e465883a055dc9c1e6ab322cb9edf406"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170409Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] + + code: post(conn, build_url(course_id, "testFolder/nestedFolder/test.png"), %{ + stacktrace: + (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 + (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 + (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 + (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 + (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 + (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 + (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 + (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 + (cadet 0.0.1) lib/cadet/assets/assets.ex:18: Cadet.Assets.Assets.upload_to_s3/4 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:42: CadetWeb.AdminAssetsController.upload/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 + (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 + (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:225: Phoenix.ConnTest.dispatch/5 + test/cadet_web/admin_controllers/admin_assets_controller_test.exs:223: (test) + + + + 19) test ok request delete file (CadetWeb.AdminAssetsControllerTest) + test/cadet_web/admin_controllers/admin_assets_controller_test.exs:144 + ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_delete_asset#1.json. + Delete the current cassette with [mix vcr.delete] and re-record. + + Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/117/testFolder/test2.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=65fc7089e1831b82570370dac1a3da6cfbd5b7a24db4e7f0e6e77d72323246c8"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170409Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] + + code: conn = delete(conn, build_url(course_id, "testFolder/test2.png")) + stacktrace: + (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 + (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 + (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 + (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 + (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 + (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 + (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 + (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 + (cadet 0.0.1) lib/cadet/assets/assets.ex:55: Cadet.Assets.Assets.delete_object/3 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:23: CadetWeb.AdminAssetsController.delete/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 + (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 + (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:225: Phoenix.ConnTest.dispatch/5 + test/cadet_web/admin_controllers/admin_assets_controller_test.exs:148: (test) + +.... + + 20) test course with custom assets_prefix index file (CadetWeb.AdminAssetsControllerTest) + test/cadet_web/admin_controllers/admin_assets_controller_test.exs:235 + ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_list_assets#2.json. + Delete the current cassette with [mix vcr.delete] and re-record. + + Request: [:get, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/?prefix=this-is-a-prefix%2FtestFolder%2F", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=54e3be18feab3eb2537292a44a21f14fb8a0ff99880af3dee7ec2f48cd3f3d1c"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170409Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] + + stacktrace: + (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 + (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 + (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 + (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 + (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 + (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 + (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 + (ex_aws 2.5.9) lib/ex_aws.ex:85: ExAws.request!/2 + (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:7: anonymous fn/4 in ExAws.S3.Lazy.stream_objects!/3 + (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:18: anonymous fn/2 in ExAws.S3.Lazy.stream_objects!/3 + (elixir 1.18.3) lib/stream.ex:1571: Stream.do_resource/5 + (elixir 1.18.3) lib/enum.ex:4515: Enum.map/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:12: CadetWeb.AdminAssetsController.index/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 + (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 + (hackney 1.25.0) :hackney.request(:get, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/?prefix=this-is-a-prefix%2FtestFolder%2F", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=54e3be18feab3eb2537292a44a21f14fb8a0ff99880af3dee7ec2f48cd3f3d1c"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170409Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]) + +.... + + 21) test ok request upload file (CadetWeb.AdminAssetsControllerTest) + test/cadet_web/admin_controllers/admin_assets_controller_test.exs:155 + ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_upload_asset#1.json. + Delete the current cassette with [mix vcr.delete] and re-record. + + Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/117/testFolder/test.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=26862231f49900869683bb7163788ff85fa166d6e89a3f192b6744b765aff03f"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170409Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] + + code: post(conn, build_url(course_id, "testFolder/test.png"), %{ + stacktrace: + (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 + (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 + (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 + (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 + (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 + (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 + (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 + (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 + (cadet 0.0.1) lib/cadet/assets/assets.ex:18: Cadet.Assets.Assets.upload_to_s3/4 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:42: CadetWeb.AdminAssetsController.upload/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 + (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 + (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 + (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 + (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:225: Phoenix.ConnTest.dispatch/5 + test/cadet_web/admin_controllers/admin_assets_controller_test.exs:160: (test) + +..01:04:09.839 [error] Team creation failed for assessment 16932 - students not enrolled in course +..01:04:09.854 [error] Team creation failed for assessment 16936 - students already assigned to teams +.01:04:09.862 [error] Cannot delete team 281 - team has submitted answers +.01:04:09.866 [error] Team creation failed for assessment 16940 - duplicate students +.01:04:09.872 [error] Team creation failed for assessment 16943 - teams exceed maximum size +.01:04:09.877 [error] Team creation failed for assessment 16944 - duplicate students +...01:04:09.898 [error] Team creation failed for assessment 16950 - students not enrolled in course +......................................01:04:13.872 [error] Team not found for assessment 17673 and user 8870 +.01:04:13.873 [error] Team not found for question 17549 and user 8870 +.....01:04:14.477 [error] Team not found for assessment 17788 and user 8904 +01:04:14.477 [error] Team not found for user 8904 in assessment 17788 +........01:04:15.357 [error] Contest assessment not found +01:04:15.357 [error] Contest assessment not found +...........01:04:16.616 [error] Cannot delete assessment 18142 - contest voting is still active +................01:04:18.044 [error] Team not found for assessment 18411 and user 9131 +01:04:18.044 [error] Team not found for user 9131 in assessment 18411 +...01:04:18.616 [error] Failed to create question for assessment: type can't be blank +question can't be blank +library can't be blank +....01:04:19.495 [error] Force update failed: Question count is different +...........................01:04:23.952 request_id=GHNTqNx1BmXkEikAAHJB [error] Failed to retrieve assessment 19245 for user 9319: forbidden - Missing Password. +01:04:23.952 request_id=GHNTqNx1BmXkEikAAHJB [error] Error fetching assessment 19245 for user 9423: Missing Password. (status: forbidden). +01:04:23.954 request_id=GHNTqNydzlLkEikAAHLB [error] Failed to retrieve assessment 19245 for user 9318: forbidden - Missing Password. +01:04:23.954 request_id=GHNTqNydzlLkEikAAHLB [error] Error fetching assessment 19245 for user 9422: Missing Password. (status: forbidden). +01:04:23.956 request_id=GHNTqNy38vDkEikAAHMB [error] Failed to retrieve assessment 19245 for user 9316: forbidden - Missing Password. +.01:04:23.956 request_id=GHNTqNy38vDkEikAAHMB [error] Error fetching assessment 19245 for user 9420: Missing Password. (status: forbidden). +...01:04:25.358 request_id=GHNTqTBRtO9Yv4sAAFlC [error] Cannot finalize submission 17026 - some questions have not been attempted +01:04:25.358 request_id=GHNTqTBRtO9Yv4sAAFlC [error] Error submitting assessment 19369 for user 9482: Some questions have not been attempted (status: bad_request). +.......01:04:27.967 request_id=GHNTqcvm2EuzRw0AAHlB [error] User record not found for user 9446 in course 38770 +...01:04:29.003 request_id=GHNTqgmUtGCHonMAAHrB [error] Cannot finalize submission 17437 - assessment has already been submitted +01:04:29.003 request_id=GHNTqgmUtGCHonMAAHrB [error] Error submitting assessment 19772 for user 9605: Assessment has already been submitted (status: forbidden). +...01:04:29.938 request_id=GHNTqkFUj-IRInoAAHwB [error] Submission not found for assessment 19897 and user 9648. +...01:04:31.015 request_id=GHNTqoF1TgfZgfgAAH2B [error] Assessment 20017 is not open for user 9676 +01:04:31.015 request_id=GHNTqoF1TgfZgfgAAH2B [error] Failed to retrieve assessment 20017 for user 9551: forbidden - Assessment not open +.01:04:31.015 request_id=GHNTqoF1TgfZgfgAAH2B [error] Error fetching assessment 20017 for user 9676: Assessment not open (status: forbidden). +01:04:31.389 request_id=GHNTqpfCPPw3T-IAAH3B [error] Assessment 20057 not found or not published for user 9562 +01:04:31.389 request_id=GHNTqpfCPPw3T-IAAH3B [error] Error fetching assessment 20057 for user 9688: Assessment not found (status: bad_request). +...........01:04:38.421 request_id=GHNTrDrvbxVNbcIAAKcB [error] Assessment 21093 is not open for user 10115. +.......01:04:43.267 request_id=GHNTrVu2y407QQsAAMRB [error] Failed to retrieve assessment 21480 for user 10155: forbidden - Invalid Password. +01:04:43.267 request_id=GHNTrVu2y407QQsAAMRB [error] Failed to unlock assessment 21480 for user 10300: Invalid Password. (status: forbidden). +01:04:43.270 request_id=GHNTrVvdee07QQsAAMSB [error] Failed to retrieve assessment 21480 for user 10154: forbidden - Invalid Password. +01:04:43.270 request_id=GHNTrVvdee07QQsAAMSB [error] Failed to unlock assessment 21480 for user 10299: Invalid Password. (status: forbidden). +01:04:43.272 request_id=GHNTrVwMwKQ7QQsAAMTB [error] Failed to retrieve assessment 21480 for user 10152: forbidden - Invalid Password. +01:04:43.272 request_id=GHNTrVwMwKQ7QQsAAMTB [error] Failed to unlock assessment 21480 for user 10297: Invalid Password. (status: forbidden). +..01:04:44.025 request_id=GHNTrYjuj3Jl6G8AAMVB [error] Submission not found for assessment 21521 and user 10333. +.........01:04:46.603 request_id=GHNTriKR2RuRJEwAAMyB [error] Submission 19212 is not in a submitted state +.01:04:46.603 request_id=GHNTriKR2RuRJEwAAMyB [error] Submission 19212 has not been submitted +.......01:04:47.098 request_id=GHNTrkAdRI3ihYoAAM-B [error] Assessment for submission 19264 is not open +...01:04:47.174 request_id=GHNTrkSVrEKsiWMAAM_B [error] User 10490 is not allowed to publish submission 19271 +.01:04:47.174 request_id=GHNTrkSVrEKsiWMAAM_B [error] User 10490 is not allowed to unpublish grading for submission 19271 +...01:04:47.260 request_id=GHNTrkmydNhSt2UAANCB [error] Grading for submission 19279 has already been published +......................01:04:48.831 request_id=GHNTrqdXIXvzkYkAANTB [error] User 10641 is not allowed to publish submission 19398 +.01:04:48.831 request_id=GHNTrqdXIXvzkYkAANTB [error] User 10641 is not allowed to publish grading for submission 19398 +01:04:48.862 request_id=GHNTrqlEC4746yAAANUB [error] Submission 19401 has already been attempted +.01:04:48.895 request_id=GHNTrqsuBJjPRD0AANVB [error] User 10654 is not allowed to unsubmit submission 19404 +............. +Finished in 92.1 seconds (3.7s async, 88.3s sync) +943 tests, 21 failures, 7 invalid diff --git a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs index 1d766f6a2..ed429bf50 100644 --- a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs @@ -257,8 +257,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "coverPicture" => assessment.cover_picture, "number" => assessment.number, "story" => assessment.story, - "reading" => assessment.reading, - "llm_assessment_prompt" => assessment.llm_assessment_prompt + "reading" => assessment.reading }, "answers" => answers @@ -268,6 +267,8 @@ defmodule CadetWeb.AdminGradingControllerTest do &case &1.question.type do :programming -> %{ + "id" => &1.id, + "prompts" => [], "question" => %{ "prepend" => &1.question.question.prepend, "postpend" => &1.question.question.postpend, @@ -314,8 +315,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "content" => &1.question.question.content, "answer" => &1.answer.code, "autogradingStatus" => Atom.to_string(&1.autograding_status), - "autogradingResults" => &1.autograding_results, - "llm_prompt" => &1.question.question.llm_prompt + "autogradingResults" => &1.autograding_results }, "ai_comments" => nil, "solution" => &1.question.question.solution, @@ -339,6 +339,8 @@ defmodule CadetWeb.AdminGradingControllerTest do :mcq -> %{ + "id" => &1.id, + "prompts" => [], "question" => %{ "type" => "#{&1.question.type}", "blocking" => &1.question.blocking, @@ -389,6 +391,8 @@ defmodule CadetWeb.AdminGradingControllerTest do :voting -> %{ + "id" => &1.id, + "prompts" => [], "question" => %{ "prepend" => &1.question.question.prepend, "solutionTemplate" => &1.question.question.template, @@ -1284,8 +1288,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "coverPicture" => assessment.cover_picture, "number" => assessment.number, "story" => assessment.story, - "reading" => assessment.reading, - "llm_assessment_prompt" => assessment.llm_assessment_prompt + "reading" => assessment.reading }, "answers" => answers @@ -1295,6 +1298,8 @@ defmodule CadetWeb.AdminGradingControllerTest do &case &1.question.type do :programming -> %{ + "id" => &1.id, + "prompts" => [], "question" => %{ "prepend" => &1.question.question.prepend, "postpend" => &1.question.question.postpend, @@ -1341,8 +1346,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "content" => &1.question.question.content, "answer" => &1.answer.code, "autogradingStatus" => Atom.to_string(&1.autograding_status), - "autogradingResults" => &1.autograding_results, - "llm_prompt" => &1.question.question.llm_prompt + "autogradingResults" => &1.autograding_results }, "ai_comments" => nil, "solution" => &1.question.question.solution, @@ -1366,6 +1370,8 @@ defmodule CadetWeb.AdminGradingControllerTest do :mcq -> %{ + "id" => &1.id, + "prompts" => [], "question" => %{ "type" => "#{&1.question.type}", "blocking" => &1.question.blocking, @@ -1416,6 +1422,8 @@ defmodule CadetWeb.AdminGradingControllerTest do :voting -> %{ + "id" => &1.id, + "prompts" => [], "question" => %{ "prepend" => &1.question.question.prepend, "solutionTemplate" => &1.question.question.template, diff --git a/test/cadet_web/controllers/assessments_controller_test.exs b/test/cadet_web/controllers/assessments_controller_test.exs index bf6e3dbd7..17e7cca72 100644 --- a/test/cadet_web/controllers/assessments_controller_test.exs +++ b/test/cadet_web/controllers/assessments_controller_test.exs @@ -371,7 +371,6 @@ defmodule CadetWeb.AssessmentsControllerTest do "solutionTemplate" => &1.question.template, "prepend" => &1.question.prepend, "postpend" => &1.question.postpend, - "llm_prompt" => &1.question.llm_prompt, "testcases" => Enum.map( &1.question.public, From 8b579337646568237186082627ced3a59df8beee Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Fri, 31 Oct 2025 01:05:44 +0800 Subject: [PATCH 70/77] rm --- test.txt | 619 ------------------------------------------------------- 1 file changed, 619 deletions(-) delete mode 100644 test.txt diff --git a/test.txt b/test.txt deleted file mode 100644 index 779ccd3d1..000000000 --- a/test.txt +++ /dev/null @@ -1,619 +0,0 @@ -Compiling 3 files (.ex) -Compiling 3 files (.ex) -Generated cadet app -cadet: generated priv/static/swagger.json - -01:03:16.715 [info] Migrations already up -Running ExUnit with seed: 184121, max_cases: 44 - -.............................[ASSESSMENT_SUBMISSION] SENDING_OUT -.....[AVENGER_BACKLOG] course-level disabled -[AVENGER_BACKLOG] SENDING_OUT -[AVENGER_BACKLOG] SENDING_OUT -. - - 1) test Manage assets delete ok file (Cadet.Assets.AssetsTest) - test/cadet/assets/assets_test.exs:31 - ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/model_delete_asset#2.json. - Delete the current cassette with [mix vcr.delete] and re-record. - - Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/1/testFolder/test.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=eb4636f36507f3438e88be438b49ac665c3dd7ee8469dcc7432853f4eb9c7ba6"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170319Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] - - code: assert Assets.delete_object(prefix(1), "testFolder", "test.png") === :ok - stacktrace: - (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 - (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 - (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 - (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 - (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 - (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 - (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 - (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 - (cadet 0.0.1) lib/cadet/assets/assets.ex:55: Cadet.Assets.Assets.delete_object/3 - test/cadet/assets/assets_test.exs:33: (test) - - - - 2) test Manage assets access another course with 0 folder (Cadet.Assets.AssetsTest) - test/cadet/assets/assets_test.exs:18 - ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/model_list_assets#2.json. - Delete the current cassette with [mix vcr.delete] and re-record. - - Request: [:get, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/?prefix=courses-dev%2F2%2FtestFolder%2F", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=3e2bfe468373c6b076b8795389ea54372078104e8433755851457ca9ae85d380"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170319Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] - - code: assert Assets.list_assets(prefix(2), "testFolder") === [] - stacktrace: - (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 - (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 - (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 - (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 - (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 - (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 - (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 - (ex_aws 2.5.9) lib/ex_aws.ex:85: ExAws.request!/2 - (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:7: anonymous fn/4 in ExAws.S3.Lazy.stream_objects!/3 - (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:18: anonymous fn/2 in ExAws.S3.Lazy.stream_objects!/3 - (elixir 1.18.3) lib/stream.ex:1571: Stream.do_resource/5 - (elixir 1.18.3) lib/enum.ex:4515: Enum.map/2 - test/cadet/assets/assets_test.exs:20: (test) - - - - 3) test Manage assets upload existing file (Cadet.Assets.AssetsTest) - test/cadet/assets/assets_test.exs:37 - ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/model_upload_asset#1.json. - Delete the current cassette with [mix vcr.delete] and re-record. - - Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/1/testFolder/test2.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=13c15ffce68a98d964248228c70eede720fe0d78fb99662f0fd89b6a187e7fb6"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170320Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] - - code: assert Assets.upload_to_s3( - stacktrace: - (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 - (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 - (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 - (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 - (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 - (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 - (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 - (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 - (cadet 0.0.1) lib/cadet/assets/assets.ex:18: Cadet.Assets.Assets.upload_to_s3/4 - test/cadet/assets/assets_test.exs:39: (test) - - - - 4) test Manage assets delete nonexistent file (Cadet.Assets.AssetsTest) - test/cadet/assets/assets_test.exs:24 - ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/model_delete_asset#1.json. - Delete the current cassette with [mix vcr.delete] and re-record. - - Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/1/testFolder/test4.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=12083bf68aeedc6ceb05f9ab4e5c7bf69034aef06b99600594755aed667473aa"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170320Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] - - code: assert Assets.delete_object(prefix(1), "testFolder", "test4.png") === - stacktrace: - (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 - (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 - (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 - (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 - (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 - (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 - (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 - (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 - (cadet 0.0.1) lib/cadet/assets/assets.ex:55: Cadet.Assets.Assets.delete_object/3 - test/cadet/assets/assets_test.exs:26: (test) - - - - 5) test Manage assets accessible folder (Cadet.Assets.AssetsTest) - test/cadet/assets/assets_test.exs:8 - ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/model_list_assets#1.json. - Delete the current cassette with [mix vcr.delete] and re-record. - - Request: [:get, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/?prefix=courses-dev%2F1%2FtestFolder%2F", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=3fa280761ada3502c164c277e7aaa3781a83db339553d456ec73e18732184bd4"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170320Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] - - code: assert Assets.list_assets(prefix(1), "testFolder") === [ - stacktrace: - (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 - (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 - (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 - (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 - (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 - (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 - (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 - (ex_aws 2.5.9) lib/ex_aws.ex:85: ExAws.request!/2 - (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:7: anonymous fn/4 in ExAws.S3.Lazy.stream_objects!/3 - (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:18: anonymous fn/2 in ExAws.S3.Lazy.stream_objects!/3 - (elixir 1.18.3) lib/stream.ex:1571: Stream.do_resource/5 - (elixir 1.18.3) lib/enum.ex:4515: Enum.map/2 - test/cadet/assets/assets_test.exs:10: (test) - - - - 6) test Manage assets upload ok file (Cadet.Assets.AssetsTest) - test/cadet/assets/assets_test.exs:49 - ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/model_upload_asset#2.json. - Delete the current cassette with [mix vcr.delete] and re-record. - - Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/1/testFolder/test1.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=5364a5f81c7020737d88523c43daa502d6337034f8953925d11c3d3d2ac0439d"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170320Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] - - code: assert Assets.upload_to_s3( - stacktrace: - (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 - (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 - (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 - (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 - (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 - (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 - (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 - (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 - (cadet 0.0.1) lib/cadet/assets/assets.ex:18: Cadet.Assets.Assets.upload_to_s3/4 - test/cadet/assets/assets_test.exs:51: (test) - -.................................................01:03:21.444 request_id=GHNTmk5EyRz9IDkAADfC [error] Failed to create course for user 7361: viewable is invalid -01:03:21.444 request_id=GHNTmk5EyRz9IDkAADfC [error] Invalid parameters provided by user 7361 while creating a course. -..01:03:21.456 request_id=GHNTmk96Kd29nP4AADgC [error] User 7363 has exceeded the limit of 5 admin courses. -.01:03:21.459 request_id=GHNTmk-pD6MfhdcAABMB [error] Failed to create course for user 7364: source_chapter source chapter and source variant must be present together; can't be blank -.01:03:21.459 request_id=GHNTmk-pD6MfhdcAABMB [error] Invalid parameters provided by user 7364 while creating a course. -01:03:21.462 request_id=GHNTmk_Wkn1ssGsAABNB [error] User record not found for user 7365 in course 29701 -................................................01:03:22.199 request_id=GHNTmnvI_oX9Yp8AADsC [error] User record not found for user 7412 in course 29869 -.01:03:22.202 request_id=GHNTmnvs2Bg53agAAAtH [error] Failed to update course configuration for course 29869: source_variant is invalid -..01:03:22.206 request_id=GHNTmnwy0jC9wigAAAvH [error] User record not found for user 7415 in course 29872 -.................01:03:22.261 request_id=GHNTmn94bNTiTjEAAA6H [error] Failed to update course configuration for course 29888: source_chapter source chapter and source variant must be present together -..01:03:22.264 request_id=GHNTmn-qj43xDksAAA9E [error] User record not found for user 7433 in course 29890 -..........01:03:22.290 request_id=GHNTmoEzrqBLWjYAAD1C [error] Team creation failed for assessment 15139 - duplicate students -.....01:03:22.323 request_id=GHNTmoMo0TImw0QAABFE [error] User record not found for user 7453 in course 29925 -.01:03:22.331 request_id=GHNTmoOdXfk5HtMAABGE [error] Cannot delete team 207 - team has submitted answers -..01:03:22.345 request_id=GHNTmoSBEdG2BRoAAD9C [error] User record not found for user 7461 in course 29935 -.01:03:22.351 request_id=GHNTmoTZKrxMEfQAAD-C [error] Team creation failed for assessment 15147 - students already assigned to teams -......01:03:22.364 request_id=GHNTmoWV5G0FRCAAABIE [error] Team creation failed for assessment 15148 - teams exceed maximum size -..01:03:22.369 request_id=GHNTmoXrndIDf3YAAEDC [error] User record not found for user 7472 in course 29956 -.01:03:22.374 request_id=GHNTmoYvCJyQI1UAABJE [error] Team creation failed for assessment 15150 - students not enrolled in course -....01:03:22.377 request_id=GHNTmoZsN0WH2fMAAEHC [error] Failed to update research agreement for user 7477. Status: bad_request, Message: agreed_to_research is invalid. -.01:03:22.380 request_id=GHNTmoaNGkbGZX4AABKE [error] Failed to update game states for user 7478. Status: bad_request, Message: game_states is invalid. -.........01:03:22.465 request_id=GHNTmouj56eiGZsAABQE [error] User 7489 is not enrolled in course 29997 -.01:03:22.465 request_id=GHNTmouj56eiGZsAABQE [error] Failed to update latest viewed course for user 7489. Status: bad_request, Message: user is not in the course. -..........01:03:22.478 request_id=GHNTmoxq8LKyAZsAABRE [error] Failed to upload sourcecast for user 7492: title can't be blank -audio can't be blank -playbackData can't be blank -.............01:03:22.549 request_id=GHNTmpCh1KlKr24AABZE [error] Sourcecast 1 not found -...01:03:22.556 request_id=GHNTmpEOrZzDVVsAAEeC [error] User record not found for user 7524 in course 30042 -..........01:03:23.067 request_id=GHNTmq-OQ5ctFOsAABQB [error] User record not found for user 7542 in course 30180 -.......01:03:24.174 request_id=GHNTmvF4pV42OOcAABVB [error] Cannot delete story 78 - user not allowed to manage stories from another course -....01:03:24.191 request_id=GHNTmvJ6Qgl2BK0AAEoC [error] Cannot update story 80 - user not allowed to manage stories from another course -........01:03:24.212 request_id=GHNTmvO-cZH_0WMAAEuC [error] Failed to create story for course 30463: title can't be blank -close_at can't be blank -open_at can't be blank -filenames can't be blank -. - - 7) test GET /v2/courses/:course_id/admin/generate-comments/:submissionid/:questionid errors out when given an invalid submission (CadetWeb.AICodeAnalysisControllerTest) - test/cadet_web/controllers/ai_code_analysis_controller_test.exs:101 - ** (UndefinedFunctionError) function Cadet.Courses.Course.encrypt_llm_api_key/1 is undefined or private - stacktrace: - (cadet 0.0.1) Cadet.Courses.Course.encrypt_llm_api_key("test_key") - test/cadet_web/controllers/ai_code_analysis_controller_test.exs:12: CadetWeb.AICodeAnalysisControllerTest.__ex_unit_setup_1/1 - test/cadet_web/controllers/ai_code_analysis_controller_test.exs:3: CadetWeb.AICodeAnalysisControllerTest.__ex_unit__/2 - - - - 8) test GET /v2/courses/:course_id/admin/generate-comments/:submissionid/:questionid LLM endpoint returns an invalid response - should log errors in database (CadetWeb.AICodeAnalysisControllerTest) - test/cadet_web/controllers/ai_code_analysis_controller_test.exs:135 - ** (UndefinedFunctionError) function Cadet.Courses.Course.encrypt_llm_api_key/1 is undefined or private - stacktrace: - (cadet 0.0.1) Cadet.Courses.Course.encrypt_llm_api_key("test_key") - test/cadet_web/controllers/ai_code_analysis_controller_test.exs:12: CadetWeb.AICodeAnalysisControllerTest.__ex_unit_setup_1/1 - test/cadet_web/controllers/ai_code_analysis_controller_test.exs:3: CadetWeb.AICodeAnalysisControllerTest.__ex_unit__/2 - - - - 9) test GET /v2/courses/:course_id/admin/generate-comments/:submissionid/:questionid success with happy path, admin and staff (CadetWeb.AICodeAnalysisControllerTest) - test/cadet_web/controllers/ai_code_analysis_controller_test.exs:38 - ** (UndefinedFunctionError) function Cadet.Courses.Course.encrypt_llm_api_key/1 is undefined or private - stacktrace: - (cadet 0.0.1) Cadet.Courses.Course.encrypt_llm_api_key("test_key") - test/cadet_web/controllers/ai_code_analysis_controller_test.exs:12: CadetWeb.AICodeAnalysisControllerTest.__ex_unit_setup_1/1 - test/cadet_web/controllers/ai_code_analysis_controller_test.exs:3: CadetWeb.AICodeAnalysisControllerTest.__ex_unit__/2 - -......01:03:24.243 [error] Failed to upsert achievement eb61dfa8-51c3-4b5d-89f8-3239de478c5d: course_id can't be blank -...................................................................................................................................................................???????01:03:51.816 [error] Cannot update course 32669 - course not found -01:03:51.817 [error] Failed to update course configuration for course 32669: source_chapter is invalid -01:03:51.818 [error] Failed to update course configuration for course 32669: source_variant is invalid - - - 9) Cadet.Auth.Providers.OpenIDTest: failure on setup_all callback, all tests have been invalidated - ** (UndefinedFunctionError) function OpenIDConnect.Worker.start_link/1 is undefined (module OpenIDConnect.Worker is not available) - stacktrace: - OpenIDConnect.Worker.start_link([test: [discovery_document_uri: "http://localhost:43705/.well-known/openid-configuration", client_id: "dummy", client_secret: "dummydummy", response_type: "code", scope: "openid profile"]]) - test/cadet/auth/providers/openid/openid_test.exs:67: Cadet.Auth.Providers.OpenIDTest.__ex_unit_setup_all_0/1 - test/cadet/auth/providers/openid/openid_test.exs:1: Cadet.Auth.Providers.OpenIDTest.__ex_unit__/2 - -......................01:03:51.923 [error] Course 32701 not found -.......01:03:51.968 request_id=GHNToWn8V_7Cl2MAACsB [error] Failed to answer question 16779 - invalid parameters -01:03:51.971 request_id=GHNToWo3LEPCl2MAACvB [error] Failed to answer question 16778 - invalid parameters -01:03:51.978 request_id=GHNToWqXAmfCl2MAACwB [error] Invalid vote for question 16780 by user 8189 -......01:03:52.134 request_id=GHNToXPjPpRZHc4AADFB [error] Failed to answer question 16798 - invalid parameters -01:03:52.137 request_id=GHNToXQfkqhZHc4AADGB [error] Failed to answer question 16797 - invalid parameters -01:03:52.143 request_id=GHNToXR2X6VZHc4AADHB [error] Invalid vote for question 16799 by user 8199 -.....01:03:52.243 request_id=GHNToXpb0hjp8p8AADTB [error] Failed to answer question 16813 - invalid parameters -01:03:52.248 request_id=GHNToXqg0eXp8p8AADUB [error] Failed to answer question 16812 - invalid parameters -01:03:52.255 request_id=GHNToXsoZLHp8p8AADVB [error] Invalid vote for question 16814 by user 8207 -..........01:03:52.445 request_id=GHNToYZ1N8IJtM4AADqB [error] Failed to answer question 16845 - submission already finalized -.........01:03:52.635 request_id=GHNToZHdDoc49jkAABbE [error] Login failed for provider 'test'. -.01:03:52.668 request_id=GHNToZPlm7-ESS8AACZD [error] SAML redirect failed for provider 'saml'. -01:03:52.684 request_id=GHNToZThD4VlPLwAADvB [error] Logout request failed due to missing parameters. -01:03:52.685 request_id=GHNToZTqZN5ByxwAABcE [error] Login failed for provider 'test'. -01:03:52.685 request_id=GHNToZTz5gcz2isAABdE [error] Invalid refresh token provided. - - - 10) test GET /auth/saml_redirect success (CadetWeb.AuthControllerTest) - test/cadet_web/controllers/auth_controller_test.exs:104 - ** (RuntimeError) expected response with status 302, got: 500, with body: - "Unknown error: Invalid or nonexistent provider config" - code: assert response(conn, 302) - stacktrace: - (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:373: Phoenix.ConnTest.response/2 - test/cadet_web/controllers/auth_controller_test.exs:110: (test) - -...01:03:52.713 request_id=GHNToZaS1TPB38cAABTF [error] SAML redirect failed for provider 'saml'. - - -01:03:52.726 request_id=GHNToZdZV8GHq9IAACaD [error] Refresh request failed due to missing parameters. - 11) test GET /auth/saml_redirect missing username attribute (CadetWeb.AuthControllerTest) - test/cadet_web/controllers/auth_controller_test.exs:136 - ** (RuntimeError) expected response with status 400, got: 500, with body: - "Unknown error: Invalid or nonexistent provider config" - code: assert response(conn, 400) == "Unable to validate token: Missing username attribute!" - stacktrace: - (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:373: Phoenix.ConnTest.response/2 - test/cadet_web/controllers/auth_controller_test.exs:140: (test) - -...01:03:52.751 request_id=GHNToZjTB4u9BmkAADxB [error] Invalid refresh token provided. -.01:03:52.774 request_id=GHNToZo2S_IetKQAADVG [error] SAML redirect failed for provider 'saml'. - - - 12) test GET /auth/saml_redirect missing name attribute (CadetWeb.AuthControllerTest) - test/cadet_web/controllers/auth_controller_test.exs:129 - ** (RuntimeError) expected response with status 400, got: 500, with body: - "Unknown error: Invalid or nonexistent provider config" - code: assert response(conn, 400) == "Unable to validate token: Missing name attribute!" - stacktrace: - (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:373: Phoenix.ConnTest.response/2 - test/cadet_web/controllers/auth_controller_test.exs:133: (test) - -.01:03:52.804 request_id=GHNToZwFd-HOmCoAADyB [error] Login failed for provider 'test'. -.01:03:52.813 request_id=GHNToZySFFocBfMAADzB [error] Login request failed due to missing parameters. -.01:03:52.814 request_id=GHNToZycOvc6H20AADWG [error] Invalid token provided for logout. -.01:03:52.837 request_id=GHNToZ34blUgJNsAABVF [error] SAML redirect request failed due to missing parameters. -..01:03:52.851 request_id=GHNToZ7NKHt-8qoAABWF [error] Invalid refresh token provided. -.01:03:52.872 request_id=GHNToaAOFW-TR0wAABXF [error] Login failed for provider 'test'. -.01:03:52.912 request_id=GHNToaI1e9IXrqAAACbD [error] SAML redirect failed for provider 'saml'. - - - 13) test GET /auth/saml_redirect missing SAML assertion (CadetWeb.AuthControllerTest) - test/cadet_web/controllers/auth_controller_test.exs:122 - ** (RuntimeError) expected response with status 400, got: 500, with body: - "Unknown error: Invalid or nonexistent provider config" - code: assert response(conn, 400) == "Unable to validate token: Missing SAML assertion!" - stacktrace: - (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:373: Phoenix.ConnTest.response/2 - test/cadet_web/controllers/auth_controller_test.exs:126: (test) - -.................................01:03:53.165 [error] Failed to enroll user 8202 in course 33061: role is invalid -.................................01:03:53.451 [error] No user found in conn.assigns.current_user, please check the AssignCurrentUser plug -..01:03:53.453 [error] Rate limit of 500 exceeded for user 1 -.........01:03:53.586 request_id=GHNTocqFn5HTOpIAABoE [error] Device registration 101 not found for user 8321 -.01:03:53.592 request_id=GHNTocrtQ5n2GAwAACjD [error] Device registration 103 not found for user 8322 -....01:03:53.775 request_id=GHNTodWkWcOXgEcAABpE [error] Device registration 105 not found for user 8324 -..........01:03:53.874 request_id=GHNTodvC06qXzaQAABrE [error] Device registration 107 not found for user 8327 -....01:03:53.974 request_id=GHNToeG2rp80nvkAABeF [error] Failed to rename device registration 108: title can't be blank -...............01:03:54.085 request_id=GHNToegYNL-spbgAAEBB [error] User 8364 is not authorized to access conversation 9. -.01:03:54.085 request_id=GHNToegYNL-spbgAAEBB [error] Conversation not found for user 8364, conversation 9. -01:03:54.089 request_id=GHNToeiMruNxmcQAABsE [error] Conversation -1 not found for user 8366. -.01:03:54.089 request_id=GHNToeiMruNxmcQAABsE [error] Conversation not found for user 8366, conversation -1. -01:03:54.246 request_id=GHNTofHfN5t0i_EAAECB [error] User 8367 is not authorized to access conversation 10. -01:03:54.246 request_id=GHNTofHfN5t0i_EAAECB [error] Conversation not found for user 8367, conversation 10. -...01:03:54.439 request_id=GHNTof172nihNroAABgF [error] Section is missing for user 8370. -..01:03:54.444 request_id=GHNTof3CqxMzFHAAAEDB [error] Message too long for user 8371. Length: 1001. -.............................................01:04:08.228 [error] Cannot update story 86 - story not found -..01:04:08.231 [error] Cannot delete story 88 - user not allowed to manage stories from another course -.01:04:08.233 [error] Cannot delete story 90 - story not found -....01:04:08.239 [error] Cannot update story 96 - user not allowed to manage stories from another course -....... - - 14) test course with custom assets_prefix upload file (CadetWeb.AdminAssetsControllerTest) - test/cadet_web/admin_controllers/admin_assets_controller_test.exs:262 - ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_upload_asset#3.json. - Delete the current cassette with [mix vcr.delete] and re-record. - - Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/this-is-a-prefix/testFolder/nestedFolder/test.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=3b74e4c99c27399a00df997de8e7f7cb7b51e3000eea2b45d189e7300fb0414a"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170408Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] - - code: post(conn, build_url(course_id, "testFolder/nestedFolder/test.png"), %{ - stacktrace: - (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 - (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 - (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 - (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 - (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 - (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 - (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 - (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 - (cadet 0.0.1) lib/cadet/assets/assets.ex:18: Cadet.Assets.Assets.upload_to_s3/4 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:42: CadetWeb.AdminAssetsController.upload/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 - (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 - (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:225: Phoenix.ConnTest.dispatch/5 - test/cadet_web/admin_controllers/admin_assets_controller_test.exs:269: (test) - -. - - 15) test ok request index file (CadetWeb.AdminAssetsControllerTest) - test/cadet_web/admin_controllers/admin_assets_controller_test.exs:132 - ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_list_assets#1.json. - Delete the current cassette with [mix vcr.delete] and re-record. - - Request: [:get, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/?prefix=courses-dev%2F117%2FtestFolder%2F", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=83cd5736d311d1ece755eb34f0a86ef7d6f8a5d105191173d8a9cbb293794b62"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170408Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] - - stacktrace: - (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 - (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 - (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 - (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 - (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 - (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 - (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 - (ex_aws 2.5.9) lib/ex_aws.ex:85: ExAws.request!/2 - (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:7: anonymous fn/4 in ExAws.S3.Lazy.stream_objects!/3 - (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:18: anonymous fn/2 in ExAws.S3.Lazy.stream_objects!/3 - (elixir 1.18.3) lib/stream.ex:1571: Stream.do_resource/5 - (elixir 1.18.3) lib/enum.ex:4515: Enum.map/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:12: CadetWeb.AdminAssetsController.index/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 - (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 - (hackney 1.25.0) :hackney.request(:get, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/?prefix=courses-dev%2F117%2FtestFolder%2F", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=83cd5736d311d1ece755eb34f0a86ef7d6f8a5d105191173d8a9cbb293794b62"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170408Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]) - - - - 16) test course with custom assets_prefix delete file (CadetWeb.AdminAssetsControllerTest) - test/cadet_web/admin_controllers/admin_assets_controller_test.exs:249 - ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_delete_asset#3.json. - Delete the current cassette with [mix vcr.delete] and re-record. - - Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/this-is-a-prefix/testFolder/nestedFolder/test2.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=3ad6957b6c3a2a4891a6dc436a9e46b8355f4c0173f097ddf647bac419ef8eb0"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170408Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] - - code: conn = delete(conn, build_url(course_id, "testFolder/nestedFolder/test2.png")) - stacktrace: - (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 - (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 - (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 - (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 - (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 - (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 - (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 - (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 - (cadet 0.0.1) lib/cadet/assets/assets.ex:55: Cadet.Assets.Assets.delete_object/3 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:23: CadetWeb.AdminAssetsController.delete/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 - (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 - (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:225: Phoenix.ConnTest.dispatch/5 - test/cadet_web/admin_controllers/admin_assets_controller_test.exs:255: (test) - -.. - - 17) test nested filename request delete file (CadetWeb.AdminAssetsControllerTest) - test/cadet_web/admin_controllers/admin_assets_controller_test.exs:207 - ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_delete_asset#2.json. - Delete the current cassette with [mix vcr.delete] and re-record. - - Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/117/testFolder/nestedFolder/test2.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=f816c2b657d12b86c0030828709c8f725398e9af7eb25bf3b797b24ee7331950"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170409Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] - - code: conn = delete(conn, build_url(course_id, "testFolder/nestedFolder/test2.png")) - stacktrace: - (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 - (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 - (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 - (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 - (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 - (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 - (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 - (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 - (cadet 0.0.1) lib/cadet/assets/assets.ex:55: Cadet.Assets.Assets.delete_object/3 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:23: CadetWeb.AdminAssetsController.delete/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 - (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 - (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:225: Phoenix.ConnTest.dispatch/5 - test/cadet_web/admin_controllers/admin_assets_controller_test.exs:211: (test) - - - - 18) test nested filename request upload file (CadetWeb.AdminAssetsControllerTest) - test/cadet_web/admin_controllers/admin_assets_controller_test.exs:218 - ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_upload_asset#2.json. - Delete the current cassette with [mix vcr.delete] and re-record. - - Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/117/testFolder/nestedFolder/test.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=b276e93ad753e86da3b5d8b0b43472e3e465883a055dc9c1e6ab322cb9edf406"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170409Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] - - code: post(conn, build_url(course_id, "testFolder/nestedFolder/test.png"), %{ - stacktrace: - (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 - (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 - (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 - (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 - (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 - (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 - (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 - (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 - (cadet 0.0.1) lib/cadet/assets/assets.ex:18: Cadet.Assets.Assets.upload_to_s3/4 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:42: CadetWeb.AdminAssetsController.upload/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 - (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 - (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:225: Phoenix.ConnTest.dispatch/5 - test/cadet_web/admin_controllers/admin_assets_controller_test.exs:223: (test) - - - - 19) test ok request delete file (CadetWeb.AdminAssetsControllerTest) - test/cadet_web/admin_controllers/admin_assets_controller_test.exs:144 - ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_delete_asset#1.json. - Delete the current cassette with [mix vcr.delete] and re-record. - - Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/117/testFolder/test2.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=65fc7089e1831b82570370dac1a3da6cfbd5b7a24db4e7f0e6e77d72323246c8"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170409Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] - - code: conn = delete(conn, build_url(course_id, "testFolder/test2.png")) - stacktrace: - (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 - (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 - (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 - (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 - (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 - (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 - (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 - (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 - (cadet 0.0.1) lib/cadet/assets/assets.ex:55: Cadet.Assets.Assets.delete_object/3 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:23: CadetWeb.AdminAssetsController.delete/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 - (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 - (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:225: Phoenix.ConnTest.dispatch/5 - test/cadet_web/admin_controllers/admin_assets_controller_test.exs:148: (test) - -.... - - 20) test course with custom assets_prefix index file (CadetWeb.AdminAssetsControllerTest) - test/cadet_web/admin_controllers/admin_assets_controller_test.exs:235 - ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_list_assets#2.json. - Delete the current cassette with [mix vcr.delete] and re-record. - - Request: [:get, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/?prefix=this-is-a-prefix%2FtestFolder%2F", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=54e3be18feab3eb2537292a44a21f14fb8a0ff99880af3dee7ec2f48cd3f3d1c"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170409Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] - - stacktrace: - (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 - (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 - (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 - (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 - (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 - (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 - (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 - (ex_aws 2.5.9) lib/ex_aws.ex:85: ExAws.request!/2 - (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:7: anonymous fn/4 in ExAws.S3.Lazy.stream_objects!/3 - (ex_aws_s3 2.4.0) lib/ex_aws/s3/lazy.ex:18: anonymous fn/2 in ExAws.S3.Lazy.stream_objects!/3 - (elixir 1.18.3) lib/stream.ex:1571: Stream.do_resource/5 - (elixir 1.18.3) lib/enum.ex:4515: Enum.map/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:12: CadetWeb.AdminAssetsController.index/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 - (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 - (hackney 1.25.0) :hackney.request(:get, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/?prefix=this-is-a-prefix%2FtestFolder%2F", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=54e3be18feab3eb2537292a44a21f14fb8a0ff99880af3dee7ec2f48cd3f3d1c"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170409Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]) - -.... - - 21) test ok request upload file (CadetWeb.AdminAssetsControllerTest) - test/cadet_web/admin_controllers/admin_assets_controller_test.exs:155 - ** (ExVCR.RequestNotMatchError) Request did not match with any one in the current cassette: test/fixtures/vcr_cassettes/aws/controller_upload_asset#1.json. - Delete the current cassette with [mix vcr.delete] and re-record. - - Request: [:head, "https://s3.ap-southeast-1.amazonaws.com/source-academy-assets/courses-dev/117/testFolder/test.png", [{"Authorization", "AWS4-HMAC-SHA256 Credential=hello/20251030/ap-southeast-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=26862231f49900869683bb7163788ff85fa166d6e89a3f192b6744b765aff03f"}, {"host", "s3.ap-southeast-1.amazonaws.com"}, {"x-amz-date", "20251030T170409Z"}, {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}], "", [:with_body, {:recv_timeout, 660000}]] - - code: post(conn, build_url(course_id, "testFolder/test.png"), %{ - stacktrace: - (exvcr 0.16.0) lib/exvcr/handler.ex:304: ExVCR.Handler.raise_error_if_cassette_already_exists/2 - (exvcr 0.16.0) lib/exvcr/handler.ex:234: ExVCR.Handler.get_response_from_server/3 - (ex_aws 2.5.9) lib/ex_aws/request/hackney.ex:21: ExAws.Request.Hackney.request/5 - (ex_aws 2.5.9) lib/ex_aws/request.ex:115: anonymous fn/6 in ExAws.Request.do_request/7 - (telemetry 1.3.0) /home/tkai/source-academy/backend/deps/telemetry/src/telemetry.erl:324: :telemetry.span/3 - (ex_aws 2.5.9) lib/ex_aws/request.ex:38: ExAws.Request.request_and_retry/7 - (ex_aws 2.5.9) lib/ex_aws/operation/s3.ex:40: ExAws.Operation.ExAws.Operation.S3.perform/2 - (cadet 0.0.1) lib/cadet/assets/assets.ex:72: Cadet.Assets.Assets.object_exists?/3 - (cadet 0.0.1) lib/cadet/assets/assets.ex:18: Cadet.Assets.Assets.upload_to_s3/4 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:42: CadetWeb.AdminAssetsController.upload/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.action/2 - (cadet 0.0.1) lib/cadet_web/admin_controllers/admin_assets_controller.ex:1: CadetWeb.AdminAssetsController.phoenix_controller_pipeline/2 - (phoenix 1.8.1) lib/phoenix/router.ex:416: Phoenix.Router.__call__/5 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.plug_builder_call/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint."call (overridable 3)"/2 - (cadet 0.0.1) lib/cadet_web/endpoint.ex:1: CadetWeb.Endpoint.call/2 - (phoenix 1.8.1) lib/phoenix/test/conn_test.ex:225: Phoenix.ConnTest.dispatch/5 - test/cadet_web/admin_controllers/admin_assets_controller_test.exs:160: (test) - -..01:04:09.839 [error] Team creation failed for assessment 16932 - students not enrolled in course -..01:04:09.854 [error] Team creation failed for assessment 16936 - students already assigned to teams -.01:04:09.862 [error] Cannot delete team 281 - team has submitted answers -.01:04:09.866 [error] Team creation failed for assessment 16940 - duplicate students -.01:04:09.872 [error] Team creation failed for assessment 16943 - teams exceed maximum size -.01:04:09.877 [error] Team creation failed for assessment 16944 - duplicate students -...01:04:09.898 [error] Team creation failed for assessment 16950 - students not enrolled in course -......................................01:04:13.872 [error] Team not found for assessment 17673 and user 8870 -.01:04:13.873 [error] Team not found for question 17549 and user 8870 -.....01:04:14.477 [error] Team not found for assessment 17788 and user 8904 -01:04:14.477 [error] Team not found for user 8904 in assessment 17788 -........01:04:15.357 [error] Contest assessment not found -01:04:15.357 [error] Contest assessment not found -...........01:04:16.616 [error] Cannot delete assessment 18142 - contest voting is still active -................01:04:18.044 [error] Team not found for assessment 18411 and user 9131 -01:04:18.044 [error] Team not found for user 9131 in assessment 18411 -...01:04:18.616 [error] Failed to create question for assessment: type can't be blank -question can't be blank -library can't be blank -....01:04:19.495 [error] Force update failed: Question count is different -...........................01:04:23.952 request_id=GHNTqNx1BmXkEikAAHJB [error] Failed to retrieve assessment 19245 for user 9319: forbidden - Missing Password. -01:04:23.952 request_id=GHNTqNx1BmXkEikAAHJB [error] Error fetching assessment 19245 for user 9423: Missing Password. (status: forbidden). -01:04:23.954 request_id=GHNTqNydzlLkEikAAHLB [error] Failed to retrieve assessment 19245 for user 9318: forbidden - Missing Password. -01:04:23.954 request_id=GHNTqNydzlLkEikAAHLB [error] Error fetching assessment 19245 for user 9422: Missing Password. (status: forbidden). -01:04:23.956 request_id=GHNTqNy38vDkEikAAHMB [error] Failed to retrieve assessment 19245 for user 9316: forbidden - Missing Password. -.01:04:23.956 request_id=GHNTqNy38vDkEikAAHMB [error] Error fetching assessment 19245 for user 9420: Missing Password. (status: forbidden). -...01:04:25.358 request_id=GHNTqTBRtO9Yv4sAAFlC [error] Cannot finalize submission 17026 - some questions have not been attempted -01:04:25.358 request_id=GHNTqTBRtO9Yv4sAAFlC [error] Error submitting assessment 19369 for user 9482: Some questions have not been attempted (status: bad_request). -.......01:04:27.967 request_id=GHNTqcvm2EuzRw0AAHlB [error] User record not found for user 9446 in course 38770 -...01:04:29.003 request_id=GHNTqgmUtGCHonMAAHrB [error] Cannot finalize submission 17437 - assessment has already been submitted -01:04:29.003 request_id=GHNTqgmUtGCHonMAAHrB [error] Error submitting assessment 19772 for user 9605: Assessment has already been submitted (status: forbidden). -...01:04:29.938 request_id=GHNTqkFUj-IRInoAAHwB [error] Submission not found for assessment 19897 and user 9648. -...01:04:31.015 request_id=GHNTqoF1TgfZgfgAAH2B [error] Assessment 20017 is not open for user 9676 -01:04:31.015 request_id=GHNTqoF1TgfZgfgAAH2B [error] Failed to retrieve assessment 20017 for user 9551: forbidden - Assessment not open -.01:04:31.015 request_id=GHNTqoF1TgfZgfgAAH2B [error] Error fetching assessment 20017 for user 9676: Assessment not open (status: forbidden). -01:04:31.389 request_id=GHNTqpfCPPw3T-IAAH3B [error] Assessment 20057 not found or not published for user 9562 -01:04:31.389 request_id=GHNTqpfCPPw3T-IAAH3B [error] Error fetching assessment 20057 for user 9688: Assessment not found (status: bad_request). -...........01:04:38.421 request_id=GHNTrDrvbxVNbcIAAKcB [error] Assessment 21093 is not open for user 10115. -.......01:04:43.267 request_id=GHNTrVu2y407QQsAAMRB [error] Failed to retrieve assessment 21480 for user 10155: forbidden - Invalid Password. -01:04:43.267 request_id=GHNTrVu2y407QQsAAMRB [error] Failed to unlock assessment 21480 for user 10300: Invalid Password. (status: forbidden). -01:04:43.270 request_id=GHNTrVvdee07QQsAAMSB [error] Failed to retrieve assessment 21480 for user 10154: forbidden - Invalid Password. -01:04:43.270 request_id=GHNTrVvdee07QQsAAMSB [error] Failed to unlock assessment 21480 for user 10299: Invalid Password. (status: forbidden). -01:04:43.272 request_id=GHNTrVwMwKQ7QQsAAMTB [error] Failed to retrieve assessment 21480 for user 10152: forbidden - Invalid Password. -01:04:43.272 request_id=GHNTrVwMwKQ7QQsAAMTB [error] Failed to unlock assessment 21480 for user 10297: Invalid Password. (status: forbidden). -..01:04:44.025 request_id=GHNTrYjuj3Jl6G8AAMVB [error] Submission not found for assessment 21521 and user 10333. -.........01:04:46.603 request_id=GHNTriKR2RuRJEwAAMyB [error] Submission 19212 is not in a submitted state -.01:04:46.603 request_id=GHNTriKR2RuRJEwAAMyB [error] Submission 19212 has not been submitted -.......01:04:47.098 request_id=GHNTrkAdRI3ihYoAAM-B [error] Assessment for submission 19264 is not open -...01:04:47.174 request_id=GHNTrkSVrEKsiWMAAM_B [error] User 10490 is not allowed to publish submission 19271 -.01:04:47.174 request_id=GHNTrkSVrEKsiWMAAM_B [error] User 10490 is not allowed to unpublish grading for submission 19271 -...01:04:47.260 request_id=GHNTrkmydNhSt2UAANCB [error] Grading for submission 19279 has already been published -......................01:04:48.831 request_id=GHNTrqdXIXvzkYkAANTB [error] User 10641 is not allowed to publish submission 19398 -.01:04:48.831 request_id=GHNTrqdXIXvzkYkAANTB [error] User 10641 is not allowed to publish grading for submission 19398 -01:04:48.862 request_id=GHNTrqlEC4746yAAANUB [error] Submission 19401 has already been attempted -.01:04:48.895 request_id=GHNTrqsuBJjPRD0AANVB [error] User 10654 is not allowed to unsubmit submission 19404 -............. -Finished in 92.1 seconds (3.7s async, 88.3s sync) -943 tests, 21 failures, 7 invalid From 818d1d71edb208ad7615a9f14b4f734fe0b8a5fb Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Fri, 31 Oct 2025 01:12:17 +0800 Subject: [PATCH 71/77] Fix formatting --- lib/cadet_web/helpers/ai_comments_helpers.ex | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/cadet_web/helpers/ai_comments_helpers.ex b/lib/cadet_web/helpers/ai_comments_helpers.ex index da9d95084..dc1f0189c 100644 --- a/lib/cadet_web/helpers/ai_comments_helpers.ex +++ b/lib/cadet_web/helpers/ai_comments_helpers.ex @@ -12,13 +12,15 @@ defmodule CadetWeb.AICommentsHelpers do case Base.decode64(encrypted_key) do {:ok, decoded} -> - with [iv, tag, ciphertext] <- :binary.split(decoded, <<"|">>, [:global]) do - case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do - plain_text when is_binary(plain_text) -> {:ok, plain_text} - _ -> {:decrypt_error, :decryption_failed} - end - else - _ -> {:error, :invalid_format} + case :binary.split(decoded, <<"|">>, [:global]) do + [iv, tag, ciphertext] -> + case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do + plain_text when is_binary(plain_text) -> {:ok, plain_text} + _ -> {:decrypt_error, :decryption_failed} + end + + _ -> + {:error, :invalid_format} end _ -> From 9556c259a3baab8e94e1b630f39c1ea4e0285f07 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Fri, 31 Oct 2025 01:38:47 +0800 Subject: [PATCH 72/77] Fix test --- .../controllers/generate_ai_comments.ex | 2 +- .../ai_code_analysis_controller_test.exs | 82 ++++++------------- 2 files changed, 27 insertions(+), 57 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 8ee8ccab5..14f7ae553 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -258,7 +258,7 @@ defmodule CadetWeb.AICodeAnalysisController do answer.id, Enum.at(final_messages, 0).content, Enum.at(final_messages, 1).content, - other, + Jason.encode!(other), "Unexpected JSON shape" ) diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs index 496d949d9..c6e1ca3a4 100644 --- a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -4,12 +4,13 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do use CadetWeb.ConnCase alias Cadet.{Repo, AIComments} alias Cadet.{AIComments.AIComment, Courses.Course} + alias CadetWeb.AICommentsHelpers setup do course_with_llm = insert(:course, %{ enable_llm_grading: true, - llm_api_key: Course.encrypt_llm_api_key("test_key"), + llm_api_key: AICommentsHelpers.encrypt_llm_api_key("test_key"), llm_model: "gpt-5-mini", llm_api_url: "http://testapi.com", llm_course_level_prompt: "Example Prompt" @@ -34,7 +35,7 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do }} end - describe "GET /v2/courses/:course_id/admin/generate-comments/:submissionid/:questionid" do + describe "GET /v2/courses/:course_id/admin/generate-comments/:answer_id" do test "success with happy path, admin and staff", %{ conn: conn, admin_user: admin_user, @@ -46,42 +47,24 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do answer: answer } do # Make the API call - with_mock HTTPoison, [:passthrough], - post: fn _url, _body, _headers, _opts -> - {:ok, - %HTTPoison.Response{ - status_code: 200, - body: - Jason.encode!(%{ - "choices" => [%{"message" => %{"content" => "Comment1|||Comment2"}}] - }) - }} + with_mock OpenAI, + chat_completion: fn _input, _overrides -> + {:ok, %{:choices => [%{"message" => %{"content" => "Comment1|||Comment2"}}]}} end do conn |> sign_in(staff_user.user) - |> post( - build_url_generate_ai_comments(course_with_llm.id, new_submission.id, question.id) - ) + |> post(build_url_generate_ai_comments(course_with_llm.id, answer.id)) |> json_response(200) end - with_mock HTTPoison, [:passthrough], - post: fn _url, _body, _headers, _opts -> - {:ok, - %HTTPoison.Response{ - status_code: 200, - body: - Jason.encode!(%{ - "choices" => [%{"message" => %{"content" => "Comment1|||Comment2"}}] - }) - }} + with_mock OpenAI, + chat_completion: fn _input, _overrides -> + {:ok, %{:choices => [%{"message" => %{"content" => "Comment1|||Comment2"}}]}} end do response = conn |> sign_in(admin_user.user) - |> post( - build_url_generate_ai_comments(course_with_llm.id, new_submission.id, question.id) - ) + |> post(build_url_generate_ai_comments(course_with_llm.id, answer.id)) |> json_response(200) # Verify response @@ -92,13 +75,12 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do comments = Repo.all(AIComment) assert length(comments) > 0 latest_comment = List.first(comments) - assert latest_comment.submission_id == new_submission.id - assert latest_comment.question_id == question.id + assert latest_comment.answer_id == answer.id assert latest_comment.raw_prompt != nil assert latest_comment.answers_json != nil end - test "errors out when given an invalid submission", %{ + test "errors out when given an invalid answer id", %{ conn: conn, admin_user: admin_user, staff_user: staff_user, @@ -108,26 +90,17 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do question: question, answer: answer } do - random_submission_id = 123 + random_answer_id = 324_324 # Make the API call that should fail - with_mock HTTPoison, [:passthrough], - post: fn _url, _body, _headers, _opts -> - {:ok, - %HTTPoison.Response{ - status_code: 200, - body: - Jason.encode!(%{ - "choices" => [%{"message" => %{"content" => "Comment1|||Comment2"}}] - }) - }} + with_mock OpenAI, [:passthrough], + chat_completion: fn _input, _overrides -> + {:ok, %{:choices => [%{"message" => %{"content" => "Comment1|||Comment2"}}]}} end do response = conn |> sign_in(admin_user.user) - |> post( - build_url_generate_ai_comments(course_with_llm.id, random_submission_id, question.id) - ) + |> post(build_url_generate_ai_comments(course_with_llm.id, random_answer_id)) |> text_response(400) end end @@ -143,31 +116,28 @@ defmodule CadetWeb.AICodeAnalysisControllerTest do answer: answer } do # Make the API call that should fail - with_mock HTTPoison, [:passthrough], - post: fn _url, _body, _headers, _opts -> - {:ok, %HTTPoison.Response{status_code: 200, body: "invalid response"}} + with_mock OpenAI, [:passthrough], + chat_completion: fn _input, _overrides -> + {:ok, %{"body" => "Some unexpected response"}} end do response = conn |> sign_in(admin_user.user) - |> post( - build_url_generate_ai_comments(course_with_llm.id, new_submission.id, question.id) - ) - |> text_response(500) + |> post(build_url_generate_ai_comments(course_with_llm.id, answer.id)) + |> text_response(502) end # Verify database entry even with error comments = Repo.all(AIComment) assert length(comments) > 0 latest_comment = List.first(comments) - assert latest_comment.submission_id == new_submission.id - assert latest_comment.question_id == question.id + assert latest_comment.answer_id == answer.id assert latest_comment.raw_prompt != nil assert latest_comment.answers_json != nil end end - defp build_url_generate_ai_comments(course_id, submission_id, question_id) do - "/v2/courses/#{course_id}/admin/generate-comments/#{submission_id}/#{question_id}" + defp build_url_generate_ai_comments(course_id, answer_id) do + "/v2/courses/#{course_id}/admin/generate-comments/#{answer_id}" end end From c9a650d12c7b9c82ce79bf6f287d9424828d53e1 Mon Sep 17 00:00:00 2001 From: Richard Dominick <34370238+RichDom2185@users.noreply.github.com> Date: Fri, 31 Oct 2025 12:29:16 +0800 Subject: [PATCH 73/77] Revert some dependency changes --- mix.exs | 4 ++-- mix.lock | 8 ++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/mix.exs b/mix.exs index f28c28ce4..8120ffa23 100644 --- a/mix.exs +++ b/mix.exs @@ -67,10 +67,10 @@ defmodule Cadet.Mixfile do {:guardian, "~> 2.0"}, {:guardian_db, "~> 2.0"}, {:hackney, "~> 1.6"}, - {:httpoison, "~> 2.2.3"}, + {:httpoison, "~> 1.6", override: true}, {:jason, "~> 1.2"}, {:openai, "~> 0.6.2"}, - {:openid_connect, "~> 1.0.0"}, + {:openid_connect, "~> 0.2"}, {:phoenix, "~> 1.5"}, {:phoenix_view, "~> 2.0"}, {:phoenix_ecto, "~> 4.0"}, diff --git a/mix.lock b/mix.lock index b3c498bc7..8c33b0ab9 100644 --- a/mix.lock +++ b/mix.lock @@ -47,7 +47,6 @@ "exvcr": {:hex, :exvcr, "0.16.0", "11579f43c88ae81f57c82ce4f09e3ebda4c40117c859ed39e61a653c3a0b4ff4", [:mix], [{:exjsx, "~> 4.0", [hex: :exjsx, repo: "hexpm", optional: false]}, {:finch, "~> 0.16", [hex: :finch, repo: "hexpm", optional: true]}, {:httpoison, "~> 1.0 or ~> 2.0", [hex: :httpoison, repo: "hexpm", optional: true]}, {:httpotion, "~> 3.1", [hex: :httpotion, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:meck, "~> 0.9", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "8f576af22369942f7a1482baff1f31e2f45983cf6fac45d49d2bd2e84b4d5be8"}, "faker": {:hex, :faker, "0.18.0", "943e479319a22ea4e8e39e8e076b81c02827d9302f3d32726c5bf82f430e6e14", [:mix], [], "hexpm", "bfbdd83958d78e2788e99ec9317c4816e651ad05e24cfd1196ce5db5b3e81797"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, - "finch": {:hex, :finch, "0.20.0", "5330aefb6b010f424dcbbc4615d914e9e3deae40095e73ab0c1bb0968933cadf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2658131a74d051aabfcba936093c903b8e89da9a1b63e430bee62045fa9b2ee2"}, "gen_smtp": {:hex, :gen_smtp, "1.3.0", "62c3d91f0dcf6ce9db71bcb6881d7ad0d1d834c7f38c13fa8e952f4104a8442e", [:rebar3], [{:ranch, ">= 1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "0b73fbf069864ecbce02fe653b16d3f35fd889d0fdd4e14527675565c39d84e6"}, "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"}, "gettext": {:hex, :gettext, "0.26.2", "5978aa7b21fada6deabf1f6341ddba50bc69c999e812211903b169799208f2a8", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "aa978504bcf76511efdc22d580ba08e2279caab1066b76bb9aa81c4a1e0a32a5"}, @@ -55,8 +54,7 @@ "guardian": {:hex, :guardian, "2.4.0", "efbbb397ecca881bb548560169922fc4433a05bc98c2eb96a7ed88ede9e17d64", [:mix], [{:jose, "~> 1.11.9", [hex: :jose, repo: "hexpm", optional: false]}, {:plug, "~> 1.3.3 or ~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "5c80103a9c538fbc2505bf08421a82e8f815deba9eaedb6e734c66443154c518"}, "guardian_db": {:hex, :guardian_db, "2.1.0", "ec95a9d99cdd1e550555d09a7bb4a340d8887aad0697f594590c2fd74be02426", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:guardian, "~> 1.0 or ~> 2.0", [hex: :guardian, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.13", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "f8e7d543ac92c395f3a7fd5acbe6829faeade57d688f7562e2f0fca8f94a0d70"}, "hackney": {:hex, :hackney, "1.25.0", "390e9b83f31e5b325b9f43b76e1a785cbdb69b5b6cd4e079aa67835ded046867", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "7209bfd75fd1f42467211ff8f59ea74d6f2a9e81cbcee95a56711ee79fd6b1d4"}, - "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, - "httpoison": {:hex, :httpoison, "2.2.3", "a599d4b34004cc60678999445da53b5e653630651d4da3d14675fedc9dd34bd6", [:mix], [{:hackney, "~> 1.21", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "fa0f2e3646d3762fdc73edb532104c8619c7636a6997d20af4003da6cfc53e53"}, + "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "inch_ex": {:hex, :inch_ex, "2.1.0-rc.1", "7642a8902c0d2ed5d9b5754b2fc88fedf630500d630fc03db7caca2e92dedb36", [:mix], [{:bunt, "~> 0.2", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "4ceee988760f9382d1c1d0b93ea5875727f6071693e89a0a3c49c456ef1be75d"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, @@ -69,15 +67,13 @@ "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mimerl": {:hex, :mimerl, "1.4.0", "3882a5ca67fbbe7117ba8947f27643557adec38fa2307490c4c4207624cb213b", [:rebar3], [], "hexpm", "13af15f9f68c65884ecca3a3891d50a7b57d82152792f3e19d88650aa126b144"}, - "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, "mock": {:hex, :mock, "0.3.9", "10e44ad1f5962480c5c9b9fa779c6c63de9bd31997c8e04a853ec990a9d841af", [:mix], [{:meck, "~> 0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "9e1b244c4ca2551bb17bb8415eed89e40ee1308e0fbaed0a4fdfe3ec8a4adbd3"}, "mox": {:hex, :mox, "1.2.0", "a2cd96b4b80a3883e3100a221e8adc1b98e4c3a332a8fc434c39526babafd5b3", [:mix], [{:nimble_ownership, "~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}], "hexpm", "c7b92b3cc69ee24a7eeeaf944cd7be22013c52fcb580c1f33f50845ec821089a"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_ownership": {:hex, :nimble_ownership, "1.0.1", "f69fae0cdd451b1614364013544e66e4f5d25f36a2056a9698b793305c5aa3a6", [:mix], [], "hexpm", "3825e461025464f519f3f3e4a1f9b68c47dc151369611629ad08b636b73bb22d"}, - "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "oban": {:hex, :oban, "2.18.0", "092d20bfd3d70c7ecb70960f8548d300b54bb9937c7f2e56b388f3a9ed02ec68", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "aace1eff6f8227ae38d4274af967d96f051c2f0a5152f2ef9809dd1f97866745"}, "openai": {:hex, :openai, "0.6.2", "48ee0dc74f4d0327ebf78eaeeed8c3595e10eca88fd2a7cdd8b53473645078d6", [:mix], [{:httpoison, "~> 2.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "508c8c2937ef8627d111d9142ff9cc284d39cd0c9b8244339551ac5f5fe0e643"}, - "openid_connect": {:hex, :openid_connect, "1.0.0", "afd586d9f011cf83035dc18cc1e6cc0994b7fdf04ece91f91ba8a3f0ca5e343b", [:mix], [{:finch, "~> 0.14", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "1abc2009fad37321996b64ac4b36d8113b309b697707c0774ed6d3dbad9a5005"}, + "openid_connect": {:hex, :openid_connect, "0.2.2", "c05055363330deab39ffd89e609db6b37752f255a93802006d83b45596189c0b", [:mix], [{:httpoison, "~> 1.2", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "735769b6d592124b58edd0582554ce638524c0214cd783d8903d33357d74cc13"}, "parallel_stream": {:hex, :parallel_stream, "1.1.0", "f52f73eb344bc22de335992377413138405796e0d0ad99d995d9977ac29f1ca9", [:mix], [], "hexpm", "684fd19191aedfaf387bbabbeb8ff3c752f0220c8112eb907d797f4592d6e871"}, "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, "phoenix": {:hex, :phoenix, "1.8.1", "865473a60a979551a4879db79fbfb4503e41cd809e77c85af79716578b6a456d", [:mix], [{:bandit, "~> 1.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "84d77d2b2e77c3c7e7527099bd01ef5c8560cd149c036d6b3a40745f11cd2fb2"}, From ea3bb1f1b1e46ee80f5f47fbbcff02b8e6bf5764 Mon Sep 17 00:00:00 2001 From: Richard Dominick <34370238+RichDom2185@users.noreply.github.com> Date: Fri, 31 Oct 2025 12:37:50 +0800 Subject: [PATCH 74/77] Update actions versions --- .github/workflows/cd.yml | 2 +- .github/workflows/ci.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 0e8d69233..509c4a3bc 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -29,7 +29,7 @@ jobs: OTP_VERSION: 27.3.3 steps: - uses: rlespinasse/github-slug-action@v3.x - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Cache deps uses: actions/cache@v4 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c27c7962..b8bde325e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: # needed because the postgres container does not provide a healthcheck options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Cache deps uses: actions/cache@v4 with: From d43fe02711e950b72c4c7762621b1fc09839449b Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Fri, 31 Oct 2025 14:43:19 +0800 Subject: [PATCH 75/77] Improve encrypt + decrypt robustness --- lib/cadet_web/helpers/ai_comments_helpers.ex | 32 ++++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/cadet_web/helpers/ai_comments_helpers.ex b/lib/cadet_web/helpers/ai_comments_helpers.ex index dc1f0189c..0fae2fea8 100644 --- a/lib/cadet_web/helpers/ai_comments_helpers.ex +++ b/lib/cadet_web/helpers/ai_comments_helpers.ex @@ -10,25 +10,24 @@ defmodule CadetWeb.AICommentsHelpers do secret when is_binary(secret) and byte_size(secret) >= 16 -> key = binary_part(secret, 0, min(32, byte_size(secret))) - case Base.decode64(encrypted_key) do - {:ok, decoded} -> - case :binary.split(decoded, <<"|">>, [:global]) do - [iv, tag, ciphertext] -> - case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do - plain_text when is_binary(plain_text) -> {:ok, plain_text} - _ -> {:decrypt_error, :decryption_failed} - end - + case String.split(encrypted_key, ":", parts: 3, trim: false) do + [iv_b64, tag_b64, cipher_b64] -> + with {:ok, iv} <- Base.decode64(iv_b64), + {:ok, tag} <- Base.decode64(tag_b64), + {:ok, ciphertext} <- Base.decode64(cipher_b64) do + case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do + plain_text when is_binary(plain_text) -> {:ok, plain_text} + _ -> {:decrypt_error, :decryption_failed} + end + else _ -> - {:error, :invalid_format} + Logger.error("Failed to decode one of the components of the encrypted key") + {:decrypt_error, :invalid_format} end _ -> - Logger.error( - "Failed to decode encrypted key, is it a valid AES-256 key of 16, 24 or 32 bytes?" - ) - - {:decrypt_error, :decryption_failed} + Logger.error("Encrypted key format is invalid") + {:decrypt_error, :invalid_format} end _ -> @@ -57,7 +56,8 @@ defmodule CadetWeb.AICommentsHelpers do ) # Store both the IV, ciphertext and tag - encrypted = Base.encode64(iv <> "|" <> tag <> "|" <> ciphertext) + encrypted = + Base.encode64(iv) <> ":" <> Base.encode64(tag) <> ":" <> Base.encode64(ciphertext) else {:error, :invalid_encryption_key} end From c0c434ace6f4f5a9abcfd07e26731af2659cb674 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Fri, 31 Oct 2025 15:26:25 +0800 Subject: [PATCH 76/77] Fix dialyzer --- .../controllers/generate_ai_comments.ex | 30 +++++++------------ lib/cadet_web/helpers/ai_comments_helpers.ex | 1 + 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex index 14f7ae553..24122dd4d 100644 --- a/lib/cadet_web/controllers/generate_ai_comments.ex +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -96,25 +96,17 @@ defmodule CadetWeb.AICodeAnalysisController do {:ok, answer} <- Assessments.get_answer(answer_id_parsed) do # Get head of answers (should only be one answer for given submission # and question since we filter to only 1 question) - case answer do - nil -> - conn - |> put_status(:not_found) - |> text("No answer found for the given answer_id") - - _ -> - analyze_code( - conn, - %{ - answer: answer, - api_key: key, - llm_model: course.llm_model, - llm_api_url: course.llm_api_url, - course_prompt: course.llm_course_level_prompt, - assessment_prompt: Assessments.get_llm_assessment_prompt(answer.question_id) - } - ) - end + analyze_code( + conn, + %{ + answer: answer, + api_key: key, + llm_model: course.llm_model, + llm_api_url: course.llm_api_url, + course_prompt: course.llm_course_level_prompt, + assessment_prompt: Assessments.get_llm_assessment_prompt(answer.question_id) + } + ) else :error -> conn diff --git a/lib/cadet_web/helpers/ai_comments_helpers.ex b/lib/cadet_web/helpers/ai_comments_helpers.ex index 0fae2fea8..4e2df33c9 100644 --- a/lib/cadet_web/helpers/ai_comments_helpers.ex +++ b/lib/cadet_web/helpers/ai_comments_helpers.ex @@ -2,6 +2,7 @@ defmodule CadetWeb.AICommentsHelpers do @moduledoc """ Helper functions for Managing LLM related logic """ + require Logger def decrypt_llm_api_key(nil), do: nil From 6a141a43d619578fff7f103eadb8d58bd6a16a27 Mon Sep 17 00:00:00 2001 From: tkaixiang Date: Thu, 6 Nov 2025 01:14:23 +0800 Subject: [PATCH 77/77] Re-factor schema --- lib/cadet/ai_comments/ai_comment.ex | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex index f12fde508..64d5d4cfe 100644 --- a/lib/cadet/ai_comments/ai_comment.ex +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -18,17 +18,13 @@ defmodule Cadet.AIComments.AIComment do timestamps() end + @required_fields ~w(answer_id raw_prompt answers_json)a + @optional_fields ~w(response error final_comment)a + def changeset(ai_comment, attrs) do ai_comment - |> cast(attrs, [ - :answer_id, - :raw_prompt, - :answers_json, - :response, - :error, - :final_comment - ]) - |> validate_required([:answer_id, :raw_prompt, :answers_json]) + |> cast(attrs, @required_fields ++ @optional_fields) + |> validate_required(@required_fields) |> foreign_key_constraint(:answer_id) end end