diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 0e8d69233..509c4a3bc 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -29,7 +29,7 @@ jobs: OTP_VERSION: 27.3.3 steps: - uses: rlespinasse/github-slug-action@v3.x - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Cache deps uses: actions/cache@v4 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c27c7962..b8bde325e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: # needed because the postgres container does not provide a healthcheck options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Cache deps uses: actions/cache@v4 with: diff --git a/.gitignore b/.gitignore index c6f53010c..3ab2b5e8c 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,6 @@ erl_crash.dump # Generated lexer /src/source_lexer.erl + +# Ignore log files +/log diff --git a/config/dev.secrets.exs.example b/config/dev.secrets.exs.example index 131eb0503..6b142df34 100644 --- a/config/dev.secrets.exs.example +++ b/config/dev.secrets.exs.example @@ -98,13 +98,9 @@ config :cadet, # ws_endpoint_address: "ws://hostname:port" ] - config :openai, - # find it at https://platform.openai.com/account/api-keys - api_key: "the actual api key", - # For source academy deployment, leave this as empty string.Ingeneral could find it at https://platform.openai.com/account/org-settings under "Organization ID". - organization_key: "", - # optional, passed to [HTTPoison.Request](https://hexdocs.pm/httpoison/HTTPoison.Request.html) options - http_options: [recv_timeout: 170_0000] +config :openai, + # Input your own AES-256 encryption key here for encrypting LLM API keys of 16, 24 or 32 bytes + encryption_key: "" # config :sentry, # dsn: "https://public_key/sentry.io/somethingsomething" diff --git a/config/test.exs b/config/test.exs index 30a8f8cc0..7e9aa8f86 100644 --- a/config/test.exs +++ b/config/test.exs @@ -100,3 +100,7 @@ config :cadet, Oban, testing: :manual config :cadet, Cadet.Mailer, adapter: Bamboo.TestAdapter + +config :openai, + # Input your own AES-256 encryption key here for encrypting LLM API keys + encryption_key: "b4u7g0AyN3Tu2br9WSdZQjLMQ8bed/wgQWrH2x3qPdW8D55iv10+ySgs+bxDirWE" diff --git a/lib/cadet/ai_comments.ex b/lib/cadet/ai_comments.ex new file mode 100644 index 000000000..e37133b98 --- /dev/null +++ b/lib/cadet/ai_comments.ex @@ -0,0 +1,77 @@ +defmodule Cadet.AIComments do + @moduledoc """ + Handles operations related to AI comments, including creation, updates, and retrieval. + """ + + import Ecto.Query + alias Cadet.Repo + alias Cadet.AIComments.AIComment + + @doc """ + Creates a new AI comment log entry. + """ + def create_ai_comment(attrs \\ %{}) do + %AIComment{} + |> AIComment.changeset(attrs) + |> Repo.insert() + end + + @doc """ + Gets an AI comment by ID. + """ + def get_ai_comment(id) do + case Repo.get(AIComment, id) do + nil -> {:error, :not_found} + comment -> {:ok, comment} + end + end + + @doc """ + Retrieves the latest AI comment for a specific submission and question. + Returns `nil` if no comment exists. + """ + def get_latest_ai_comment(answer_id) do + Repo.one( + from(c in AIComment, + where: c.answer_id == ^answer_id, + order_by: [desc: c.inserted_at], + limit: 1 + ) + ) + end + + @doc """ + Updates the final comment for a specific submission and question. + Returns the most recent comment entry for that submission/question. + """ + def update_final_comment(answer_id, final_comment) do + comment = get_latest_ai_comment(answer_id) + + case comment do + nil -> + {:error, :not_found} + + _ -> + comment + |> AIComment.changeset(%{final_comment: final_comment}) + |> Repo.update() + end + end + + @doc """ + Updates an existing AI comment with new attributes. + """ + def update_ai_comment(id, attrs) do + id + |> get_ai_comment() + |> case do + {:error, :not_found} -> + {:error, :not_found} + + {:ok, comment} -> + comment + |> AIComment.changeset(attrs) + |> Repo.update() + end + end +end diff --git a/lib/cadet/ai_comments/ai_comment.ex b/lib/cadet/ai_comments/ai_comment.ex new file mode 100644 index 000000000..64d5d4cfe --- /dev/null +++ b/lib/cadet/ai_comments/ai_comment.ex @@ -0,0 +1,30 @@ +defmodule Cadet.AIComments.AIComment do + @moduledoc """ + Defines the schema and changeset for AI comments. + """ + + use Ecto.Schema + import Ecto.Changeset + + schema "ai_comment_logs" do + field(:raw_prompt, :string) + field(:answers_json, :string) + field(:response, :string) + field(:error, :string) + field(:final_comment, :string) + + belongs_to(:answer, Cadet.Assessments.Answer) + + timestamps() + end + + @required_fields ~w(answer_id raw_prompt answers_json)a + @optional_fields ~w(response error final_comment)a + + def changeset(ai_comment, attrs) do + ai_comment + |> cast(attrs, @required_fields ++ @optional_fields) + |> validate_required(@required_fields) + |> foreign_key_constraint(:answer_id) + end +end diff --git a/lib/cadet/assessments/answer.ex b/lib/cadet/assessments/answer.ex index a6794117b..b8f8695b5 100644 --- a/lib/cadet/assessments/answer.ex +++ b/lib/cadet/assessments/answer.ex @@ -10,6 +10,7 @@ defmodule Cadet.Assessments.Answer do alias Cadet.Assessments.Answer.AutogradingStatus alias Cadet.Assessments.AnswerTypes.{MCQAnswer, ProgrammingAnswer, VotingAnswer} alias Cadet.Assessments.{Question, QuestionType, Submission} + alias Cadet.AIComments.AIComment @type t :: %__MODULE__{} @@ -29,6 +30,7 @@ defmodule Cadet.Assessments.Answer do belongs_to(:grader, CourseRegistration) belongs_to(:submission, Submission) belongs_to(:question, Question) + has_many(:ai_comments, AIComment, on_delete: :delete_all) timestamps() end diff --git a/lib/cadet/assessments/assessment.ex b/lib/cadet/assessments/assessment.ex index db9cc7514..edfce9bb6 100644 --- a/lib/cadet/assessments/assessment.ex +++ b/lib/cadet/assessments/assessment.ex @@ -36,6 +36,7 @@ defmodule Cadet.Assessments.Assessment do field(:max_team_size, :integer, default: 1) field(:has_token_counter, :boolean, default: false) field(:has_voting_features, :boolean, default: false) + field(:llm_assessment_prompt, :string, default: nil) belongs_to(:config, AssessmentConfig) belongs_to(:course, Course) @@ -46,7 +47,7 @@ defmodule Cadet.Assessments.Assessment do @required_fields ~w(title open_at close_at number course_id config_id max_team_size)a @optional_fields ~w(reading summary_short summary_long - is_published story cover_picture access password has_token_counter has_voting_features)a + is_published story cover_picture access password has_token_counter has_voting_features llm_assessment_prompt)a @optional_file_fields ~w(mission_pdf)a def changeset(assessment, params) do diff --git a/lib/cadet/assessments/assessments.ex b/lib/cadet/assessments/assessments.ex index 55b74e7c6..ed0cbfac5 100644 --- a/lib/cadet/assessments/assessments.ex +++ b/lib/cadet/assessments/assessments.ex @@ -3043,13 +3043,60 @@ defmodule Cadet.Assessments do } end + @spec get_answer(integer() | String.t()) :: + {:ok, Answer.t()} | {:error, {:bad_request, String.t()}} + def get_answer(id) when is_ecto_id(id) do + answer = + Answer + |> where(id: ^id) + # [a] are bindings (in SQL it is similar to FROM answers "AS a"), + # this line's alias is INNER JOIN ... "AS q" + |> join(:inner, [a], q in assoc(a, :question)) + |> join(:inner, [_, q], ast in assoc(q, :assessment)) + |> join(:inner, [..., ast], ac in assoc(ast, :config)) + |> join(:left, [a, ...], g in assoc(a, :grader)) + |> join(:left, [_, ..., g], gu in assoc(g, :user)) + |> join(:inner, [a, ...], s in assoc(a, :submission)) + |> join(:left, [_, ..., s], st in assoc(s, :student)) + |> join(:left, [..., st], u in assoc(st, :user)) + |> join(:left, [..., s, _, _], t in assoc(s, :team)) + |> join(:left, [..., t], tm in assoc(t, :team_members)) + |> join(:left, [..., tm], tms in assoc(tm, :student)) + |> join(:left, [..., tms], tmu in assoc(tms, :user)) + |> join(:left, [a, ...], ai in assoc(a, :ai_comments)) + |> preload([_, q, ast, ac, g, gu, s, st, u, t, tm, tms, tmu, ai], + ai_comments: ai, + question: {q, assessment: {ast, config: ac}}, + grader: {g, user: gu}, + submission: + {s, student: {st, user: u}, team: {t, team_members: {tm, student: {tms, user: tmu}}}} + ) + |> Repo.one() + + if is_nil(answer) do + {:error, {:bad_request, "Answer not found."}} + else + if answer.question.type == :voting do + empty_contest_entries = Map.put(answer.question.question, :contest_entries, []) + empty_popular_leaderboard = Map.put(empty_contest_entries, :popular_leaderboard, []) + empty_contest_leaderboard = Map.put(empty_popular_leaderboard, :contest_leaderboard, []) + question = Map.put(answer.question, :question, empty_contest_leaderboard) + Map.put(answer, :question, question) + end + + {:ok, answer} + end + end + @spec get_answers_in_submission(integer() | String.t()) :: {:ok, {[Answer.t()], Assessment.t()}} | {:error, {:bad_request, String.t()}} def get_answers_in_submission(id) when is_ecto_id(id) do - answer_query = + base_query = Answer |> where(submission_id: ^id) + # [a] are bindings (in SQL it is similar to FROM answers "AS a"), + # this line's alias is INNER JOIN ... "AS q" |> join(:inner, [a], q in assoc(a, :question)) |> join(:inner, [_, q], ast in assoc(q, :assessment)) |> join(:inner, [..., ast], ac in assoc(ast, :config)) @@ -3062,15 +3109,17 @@ defmodule Cadet.Assessments do |> join(:left, [..., t], tm in assoc(t, :team_members)) |> join(:left, [..., tm], tms in assoc(tm, :student)) |> join(:left, [..., tms], tmu in assoc(tms, :user)) - |> preload([_, q, ast, ac, g, gu, s, st, u, t, tm, tms, tmu], + |> join(:left, [a, ...], ai in assoc(a, :ai_comments)) + |> preload([_, q, ast, ac, g, gu, s, st, u, t, tm, tms, tmu, ai], question: {q, assessment: {ast, config: ac}}, grader: {g, user: gu}, submission: - {s, student: {st, user: u}, team: {t, team_members: {tm, student: {tms, user: tmu}}}} + {s, student: {st, user: u}, team: {t, team_members: {tm, student: {tms, user: tmu}}}}, + ai_comments: ai ) answers = - answer_query + base_query |> Repo.all() |> Enum.sort_by(& &1.question.display_order) |> Enum.map(fn ans -> @@ -3544,4 +3593,15 @@ defmodule Cadet.Assessments do end) end end + + def get_llm_assessment_prompt(question_id) do + query = + from(q in Question, + where: q.id == ^question_id, + join: a in assoc(q, :assessment), + select: a.llm_assessment_prompt + ) + + Repo.one(query) + end end diff --git a/lib/cadet/assessments/question_types/programming_question.ex b/lib/cadet/assessments/question_types/programming_question.ex index a39625de3..245e8ef11 100644 --- a/lib/cadet/assessments/question_types/programming_question.ex +++ b/lib/cadet/assessments/question_types/programming_question.ex @@ -13,13 +13,14 @@ defmodule Cadet.Assessments.QuestionTypes.ProgrammingQuestion do field(:template, :string) field(:postpend, :string, default: "") field(:solution, :string) + field(:llm_prompt, :string) embeds_many(:public, Testcase) embeds_many(:opaque, Testcase) embeds_many(:secret, Testcase) end @required_fields ~w(content template)a - @optional_fields ~w(solution prepend postpend)a + @optional_fields ~w(solution prepend postpend llm_prompt)a def changeset(question, params \\ %{}) do question diff --git a/lib/cadet/courses/course.ex b/lib/cadet/courses/course.ex index 7ddd80a49..b8a113be1 100644 --- a/lib/cadet/courses/course.ex +++ b/lib/cadet/courses/course.ex @@ -5,6 +5,7 @@ defmodule Cadet.Courses.Course do use Cadet, :model alias Cadet.Courses.AssessmentConfig + alias CadetWeb.AICommentsHelpers @type t :: %__MODULE__{ course_name: String.t(), @@ -18,6 +19,11 @@ defmodule Cadet.Courses.Course do top_contest_leaderboard_display: integer(), enable_sourcecast: boolean(), enable_stories: boolean(), + enable_llm_grading: boolean(), + llm_api_key: String.t() | nil, + llm_model: String.t() | nil, + llm_api_url: String.t() | nil, + llm_course_level_prompt: String.t() | nil, source_chapter: integer(), source_variant: String.t(), module_help_text: String.t(), @@ -36,6 +42,11 @@ defmodule Cadet.Courses.Course do field(:top_contest_leaderboard_display, :integer, default: 10) field(:enable_sourcecast, :boolean, default: true) field(:enable_stories, :boolean, default: false) + field(:enable_llm_grading, :boolean, default: false) + field(:llm_api_key, :string, default: nil) + field(:llm_model, :string, default: nil) + field(:llm_api_url, :string, default: nil) + field(:llm_course_level_prompt, :string, default: nil) field(:source_chapter, :integer) field(:source_variant, :string) field(:module_help_text, :string) @@ -50,13 +61,39 @@ defmodule Cadet.Courses.Course do @required_fields ~w(course_name viewable enable_game enable_achievements enable_overall_leaderboard enable_contest_leaderboard top_leaderboard_display top_contest_leaderboard_display enable_sourcecast enable_stories source_chapter source_variant)a - @optional_fields ~w(course_short_name module_help_text)a + @optional_fields ~w(course_short_name module_help_text enable_llm_grading llm_api_key llm_model llm_api_url llm_course_level_prompt)a def changeset(course, params) do course |> cast(params, @required_fields ++ @optional_fields) |> validate_required(@required_fields) |> validate_sublanguage_combination(params) + |> put_encrypted_llm_api_key() + end + + def put_encrypted_llm_api_key(changeset) do + if llm_api_key = get_change(changeset, :llm_api_key) do + if is_binary(llm_api_key) and llm_api_key != "" do + encrypted = AICommentsHelpers.encrypt_llm_api_key(llm_api_key) + + case encrypted do + {:error, :invalid_encryption_key} -> + add_error( + changeset, + :llm_api_key, + "encryption key is not configured properly, cannot store LLM API key" + ) + + encrypted -> + put_change(changeset, :llm_api_key, encrypted) + end + else + # If empty string or nil is provided, don't encrypt but don't add error + changeset + end + else + changeset + end end # Validates combination of Source chapter and variant diff --git a/lib/cadet/jobs/xml_parser.ex b/lib/cadet/jobs/xml_parser.ex index 53ed83d80..206ed8c60 100644 --- a/lib/cadet/jobs/xml_parser.ex +++ b/lib/cadet/jobs/xml_parser.ex @@ -98,6 +98,8 @@ defmodule Cadet.Updater.XMLParser do reading: ~x"//READING/text()" |> transform_by(&process_charlist/1), summary_short: ~x"//WEBSUMMARY/text()" |> transform_by(&process_charlist/1), summary_long: ~x"./TEXT/text()" |> transform_by(&process_charlist/1), + llm_assessment_prompt: + ~x"./LLM_ASSESSMENT_PROMPT/text()" |> transform_by(&process_charlist/1), password: ~x"//PASSWORD/text()"so |> transform_by(&process_charlist/1) ) |> Map.put(:is_published, false) @@ -202,7 +204,8 @@ defmodule Cadet.Updater.XMLParser do prepend: ~x"./SNIPPET/PREPEND/text()" |> transform_by(&process_charlist/1), template: ~x"./SNIPPET/TEMPLATE/text()" |> transform_by(&process_charlist/1), postpend: ~x"./SNIPPET/POSTPEND/text()" |> transform_by(&process_charlist/1), - solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1) + solution: ~x"./SNIPPET/SOLUTION/text()" |> transform_by(&process_charlist/1), + llm_prompt: ~x"./LLM_GRADING_PROMPT/text()" |> transform_by(&process_charlist/1) ), entity |> xmap( diff --git a/lib/cadet_web/admin_controllers/admin_courses_controller.ex b/lib/cadet_web/admin_controllers/admin_courses_controller.ex index 2d32d20f0..2a99dee86 100644 --- a/lib/cadet_web/admin_controllers/admin_courses_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_courses_controller.ex @@ -112,6 +112,8 @@ defmodule CadetWeb.AdminCoursesController do top_contest_leaderboard_display(:body, :integer, "Top Contest Leaderboard Display") enable_sourcecast(:body, :boolean, "Enable sourcecast") enable_stories(:body, :boolean, "Enable stories") + enable_llm_grading(:body, :boolean, "Enable LLM grading") + llm_api_key(:body, :string, "OpenAI API key for this course") sublanguage(:body, Schema.ref(:AdminSublanguage), "sublanguage object") module_help_text(:body, :string, "Module help text") end diff --git a/lib/cadet_web/admin_controllers/admin_grading_controller.ex b/lib/cadet_web/admin_controllers/admin_grading_controller.ex index 9e7507bd7..564988b9f 100644 --- a/lib/cadet_web/admin_controllers/admin_grading_controller.ex +++ b/lib/cadet_web/admin_controllers/admin_grading_controller.ex @@ -2,7 +2,7 @@ defmodule CadetWeb.AdminGradingController do use CadetWeb, :controller use PhoenixSwagger - alias Cadet.Assessments + alias Cadet.{Assessments, Courses} @doc """ # Query Parameters @@ -72,7 +72,15 @@ defmodule CadetWeb.AdminGradingController do def show(conn, %{"submissionid" => submission_id}) when is_ecto_id(submission_id) do case Assessments.get_answers_in_submission(submission_id) do {:ok, {answers, assessment}} -> - render(conn, "show.json", answers: answers, assessment: assessment) + case Courses.get_course_config(assessment.course_id) do + {:ok, course} -> + render(conn, "show.json", course: course, answers: answers, assessment: assessment) + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end {:error, {status, message}} -> conn diff --git a/lib/cadet_web/admin_views/admin_grading_view.ex b/lib/cadet_web/admin_views/admin_grading_view.ex index 313a27f05..f6a90d17d 100644 --- a/lib/cadet_web/admin_views/admin_grading_view.ex +++ b/lib/cadet_web/admin_views/admin_grading_view.ex @@ -2,12 +2,19 @@ defmodule CadetWeb.AdminGradingView do use CadetWeb, :view import CadetWeb.AssessmentsHelpers + alias CadetWeb.AICodeAnalysisController - def render("show.json", %{answers: answers, assessment: assessment}) do + def render("show.json", %{course: course, answers: answers, assessment: assessment}) do %{ assessment: render_one(assessment, CadetWeb.AdminGradingView, "assessment.json", as: :assessment), - answers: render_many(answers, CadetWeb.AdminGradingView, "grading_info.json", as: :answer) + answers: + render_many(answers, CadetWeb.AdminGradingView, "grading_info.json", + as: :answer, + course: course, + assessment: assessment + ), + enable_llm_grading: course.enable_llm_grading } end @@ -147,11 +154,14 @@ defmodule CadetWeb.AdminGradingView do } end - def render("grading_info.json", %{answer: answer}) do + def render("grading_info.json", %{answer: answer, course: course, assessment: assessment}) do transform_map_for_view(answer, %{ + id: & &1.id, + prompts: &build_prompts(&1, course, assessment), + ai_comments: &extract_ai_comments_per_answer(&1.id, &1.ai_comments), student: &extract_student_data(&1.submission.student), team: &extract_team_data(&1.submission.team), - question: &build_grading_question/1, + question: &build_grading_question(&1, course, assessment), solution: &(&1.question.question["solution"] || ""), grade: &build_grade/1 }) @@ -161,6 +171,18 @@ defmodule CadetWeb.AdminGradingView do %{cols: cols, rows: summary} end + defp extract_ai_comments_per_answer(id, ai_comments) do + matching_comment = + ai_comments + # Equivalent to fn comment -> comment.question_id == question_id end + |> Enum.find(&(&1.answer_id == id)) + + case matching_comment do + nil -> nil + comment -> %{response: comment.response, insertedAt: comment.inserted_at} + end + end + defp extract_student_data(nil), do: %{} defp extract_student_data(student) do @@ -190,14 +212,26 @@ defmodule CadetWeb.AdminGradingView do end end - defp build_grading_question(answer) do - %{question: answer.question} + defp build_grading_question(answer, course, assessment) do + %{question: answer.question |> Map.delete(:llm_prompt)} |> build_question_by_question_config(true) |> Map.put(:answer, answer.answer["code"] || answer.answer["choice_id"]) |> Map.put(:autogradingStatus, answer.autograding_status) |> Map.put(:autogradingResults, answer.autograding_results) end + defp build_prompts(answer, course, assessment) do + if course.enable_llm_grading do + AICodeAnalysisController.create_final_messages( + course.llm_course_level_prompt, + assessment.llm_assessment_prompt, + answer + ) + else + [] + end + end + defp build_grade(answer = %{grader: grader}) do transform_map_for_view(answer, %{ grader: grader_builder(grader), diff --git a/lib/cadet_web/controllers/courses_controller.ex b/lib/cadet_web/controllers/courses_controller.ex index 87e4f62ba..3ad4c415d 100644 --- a/lib/cadet_web/controllers/courses_controller.ex +++ b/lib/cadet_web/controllers/courses_controller.ex @@ -83,6 +83,18 @@ defmodule CadetWeb.CoursesController do enable_sourcecast(:body, :boolean, "Enable sourcecast", required: true) enable_stories(:body, :boolean, "Enable stories", required: true) + enable_llm_grading(:body, :boolean, "Enable LLM grading", required: false) + llm_api_key(:body, :string, "OpenAI API key for this course", required: false) + llm_model(:body, :string, "LLM model to be used for this course", required: false) + llm_api_url(:body, :string, "LLM API URL to be used for this course", required: false) + + llm_course_level_prompt( + :body, + :string, + "LLM course level prompt to be used for this course", + required: false + ) + source_chapter(:body, :number, "Default source chapter", required: true) source_variant(:body, Schema.ref(:SourceVariant), "Default source variant name", @@ -132,6 +144,15 @@ defmodule CadetWeb.CoursesController do enable_sourcecast(:boolean, "Enable sourcecast", required: true) enable_stories(:boolean, "Enable stories", required: true) + enable_llm_grading(:boolean, "Enable LLM grading", required: false) + llm_api_key(:string, "OpenAI API key for this course", required: false) + llm_model(:string, "LLM model to be used for this course", required: false) + llm_api_url(:string, "LLM API URL to be used for this course", required: false) + + llm_course_level_prompt(:string, "LLM course level prompt to be used for this course", + required: false + ) + source_chapter(:integer, "Source Chapter number from 1 to 4", required: true) source_variant(Schema.ref(:SourceVariant), "Source Variant name", required: true) module_help_text(:string, "Module help text", required: true) @@ -150,6 +171,12 @@ defmodule CadetWeb.CoursesController do top_contest_leaderboard_display: 10, enable_sourcecast: true, enable_stories: false, + enable_llm_grading: false, + llm_api_key: "sk-1234567890", + llm_model: "gpt-4", + llm_api_url: "https://api.openai.com/v1/chat/completions", + llm_course_level_prompt: + "You are a helpful teaching assistant for an introductory programming course", source_chapter: 1, source_variant: "default", module_help_text: "Help text", diff --git a/lib/cadet_web/controllers/generate_ai_comments.ex b/lib/cadet_web/controllers/generate_ai_comments.ex new file mode 100644 index 000000000..24122dd4d --- /dev/null +++ b/lib/cadet_web/controllers/generate_ai_comments.ex @@ -0,0 +1,354 @@ +defmodule CadetWeb.AICodeAnalysisController do + use CadetWeb, :controller + use PhoenixSwagger + require HTTPoison + require Logger + + alias Cadet.{Assessments, AIComments, Courses} + alias CadetWeb.{AICodeAnalysisController, AICommentsHelpers} + + # For logging outputs to both database and file + defp save_comment(answer_id, raw_prompt, answers_json, response, error \\ nil) do + # Log to database + attrs = %{ + answer_id: answer_id, + raw_prompt: raw_prompt, + answers_json: answers_json, + response: response, + error: error + } + + # Check if a comment already exists for the given answer_id + case AIComments.get_latest_ai_comment(answer_id) do + nil -> + # If no existing comment, create a new one + case AIComments.create_ai_comment(attrs) do + {:ok, comment} -> + {:ok, comment} + + {:error, changeset} -> + Logger.error("Failed to log AI comment to database: #{inspect(changeset.errors)}") + {:error, changeset} + end + + existing_comment -> + # Convert the existing comment struct to a map before merging + updated_attrs = Map.merge(Map.from_struct(existing_comment), attrs) + + case AIComments.update_ai_comment(existing_comment.id, updated_attrs) do + {:error, :not_found} -> + Logger.error("AI comment to update not found in database") + {:error, :not_found} + + {:ok, updated_comment} -> + {:ok, updated_comment} + + {:error, changeset} -> + Logger.error("Failed to update AI comment in database: #{inspect(changeset.errors)}") + {:error, changeset} + end + end + end + + defp check_llm_grading_parameters(llm_api_key, llm_model, llm_api_url, llm_course_level_prompt) do + cond do + is_nil(llm_model) or llm_model == "" -> + {:parameter_error, "LLM model is not configured for this course"} + + is_nil(llm_api_url) or llm_api_url == "" -> + {:parameter_error, "LLM API URL is not configured for this course"} + + is_nil(llm_course_level_prompt) or llm_course_level_prompt == "" -> + {:parameter_error, "LLM course-level prompt is not configured for this course"} + + true -> + {:ok} + end + end + + defp ensure_llm_enabled(course) do + if course.enable_llm_grading do + {:ok} + else + {:error, {:forbidden, "LLM grading is not enabled for this course"}} + end + end + + @doc """ + Fetches the question details and answers based on answer_id and generates AI-generated comments. + """ + def generate_ai_comments(conn, %{ + "answer_id" => answer_id, + "course_id" => course_id + }) + when is_ecto_id(answer_id) do + with {answer_id_parsed, ""} <- Integer.parse(answer_id), + {:ok, course} <- Courses.get_course_config(course_id), + {:ok} <- ensure_llm_enabled(course), + {:ok, key} <- AICommentsHelpers.decrypt_llm_api_key(course.llm_api_key), + {:ok} <- + check_llm_grading_parameters( + key, + course.llm_model, + course.llm_api_url, + course.llm_course_level_prompt + ), + {:ok, answer} <- Assessments.get_answer(answer_id_parsed) do + # Get head of answers (should only be one answer for given submission + # and question since we filter to only 1 question) + analyze_code( + conn, + %{ + answer: answer, + api_key: key, + llm_model: course.llm_model, + llm_api_url: course.llm_api_url, + course_prompt: course.llm_course_level_prompt, + assessment_prompt: Assessments.get_llm_assessment_prompt(answer.question_id) + } + ) + else + :error -> + conn + |> put_status(:bad_request) + |> text("Invalid question ID format") + + {:decrypt_error, err} -> + conn + |> put_status(:internal_server_error) + |> text("Failed to decrypt LLM API key") + + # Errors for check_llm_grading_parameters + {:parameter_error, error_msg} -> + conn + |> put_status(:bad_request) + |> text(error_msg) + + {:error, {status, message}} -> + conn + |> put_status(status) + |> text(message) + end + end + + defp format_student_answer(answer) do + """ + **Student Answer:** + ``` + #{answer.answer["code"] || "N/A"} + ``` + """ + end + + defp format_system_prompt(course_prompt, assessment_prompt, answer) do + "**Course Level Prompt:**\n\n" <> + (course_prompt || "") <> + "\n\n**Assessment Level Prompt:**" <> + (assessment_prompt || "") <> + "\n\n" <> + """ + **Additional Instructions for this Question:** + #{answer.question.question["llm_prompt"] || "N/A"} + + **Question:** + ``` + #{answer.question.question["content"] || "N/A"} + ``` + + **Model Solution:** + ``` + #{answer.question.question["solution"] || "N/A"} + ``` + + **Autograding Status:** #{answer.autograding_status || "N/A"} + **Autograding Results:** #{format_autograding_results(answer.autograding_results)} + + The student answer will be given below as part of the User Prompt. + """ + end + + def create_final_messages( + course_prompt, + assessment_prompt, + answer + ) do + formatted_answer = + answer + |> format_student_answer() + |> Jason.encode!() + + [ + %{role: "system", content: format_system_prompt(course_prompt, assessment_prompt, answer)}, + %{role: "user", content: formatted_answer} + ] + end + + defp format_autograding_results(nil), do: "N/A" + + defp format_autograding_results(results) when is_list(results) do + Enum.map_join(results, "; ", fn result -> + "Error: #{result["errorMessage"] || "N/A"}, Type: #{result["errorType"] || "N/A"}" + end) + end + + defp format_autograding_results(results), do: inspect(results) + + defp analyze_code( + conn, + %{ + answer: answer, + api_key: api_key, + llm_model: llm_model, + llm_api_url: llm_api_url, + course_prompt: course_prompt, + assessment_prompt: assessment_prompt + } + ) do + # Combine prompts if llm_prompt exists + final_messages = + create_final_messages( + course_prompt, + assessment_prompt, + answer + ) + + input = + [ + model: llm_model, + messages: final_messages + ] + + case OpenAI.chat_completion(input, %OpenAI.Config{ + api_url: llm_api_url, + api_key: api_key, + http_options: [ + # connect timeout + timeout: 60_000, + # response timeout + recv_timeout: 60_000 + ] + }) do + {:ok, %{choices: [%{"message" => %{"content" => content}} | _]}} -> + save_comment( + answer.id, + Enum.at(final_messages, 0).content, + Enum.at(final_messages, 1).content, + content + ) + + comments_list = String.split(content, "|||") + + filtered_comments = + Enum.filter(comments_list, fn comment -> + String.trim(comment) != "" + end) + + json(conn, %{"comments" => filtered_comments}) + + {:ok, other} -> + save_comment( + answer.id, + Enum.at(final_messages, 0).content, + Enum.at(final_messages, 1).content, + Jason.encode!(other), + "Unexpected JSON shape" + ) + + conn + |> put_status(:bad_gateway) + |> text("Unexpected response format from LLM") + + {:error, reason} -> + save_comment( + answer.id, + Enum.at(final_messages, 0).content, + Enum.at(final_messages, 1).content, + nil, + inspect(reason) + ) + + conn + |> put_status(:internal_server_error) + |> text("LLM request error: #{inspect(reason)}") + end + end + + @doc """ + Saves the final comment chosen for a submission. + """ + def save_final_comment(conn, %{ + "answer_id" => answer_id, + "comment" => comment + }) do + case AIComments.update_final_comment(answer_id, comment) do + {:ok, _updated_comment} -> + json(conn, %{"status" => "success"}) + + {:error, changeset} -> + conn + |> put_status(:unprocessable_entity) + |> text("Failed to save final comment") + end + end + + swagger_path :generate_ai_comments do + post("/courses/{course_id}/admin/generate-comments/{answer_id}") + + summary("Generate AI comments for a given submission.") + + security([%{JWT: []}]) + + consumes("application/json") + produces("application/json") + + parameters do + course_id(:path, :integer, "course id", required: true) + answer_id(:path, :integer, "answer id", required: true) + end + + response(200, "OK", Schema.ref(:GenerateAIComments)) + response(400, "Invalid or missing parameter(s) or submission and/or question not found") + response(401, "Unauthorized") + response(403, "Forbidden") + response(403, "LLM grading is not enabled for this course") + end + + swagger_path :save_final_comment do + post("/courses/{course_id}/admin/save-final-comment/{answer_id}") + + summary("Save the final comment chosen for a submission.") + + security([%{JWT: []}]) + + consumes("application/json") + produces("application/json") + + parameters do + course_id(:path, :integer, "course id", required: true) + answer_id(:path, :integer, "answer id", required: true) + comment(:body, :string, "The final comment to save", required: true) + end + + response(200, "OK", Schema.ref(:SaveFinalComment)) + response(400, "Invalid or missing parameter(s)") + response(401, "Unauthorized") + response(403, "Forbidden") + end + + def swagger_definitions do + %{ + GenerateAIComments: + swagger_schema do + properties do + comments(:string, "AI-generated comments on the submission answers") + end + end, + SaveFinalComment: + swagger_schema do + properties do + status(:string, "Status of the operation") + end + end + } + end +end diff --git a/lib/cadet_web/helpers/ai_comments_helpers.ex b/lib/cadet_web/helpers/ai_comments_helpers.ex new file mode 100644 index 000000000..4e2df33c9 --- /dev/null +++ b/lib/cadet_web/helpers/ai_comments_helpers.ex @@ -0,0 +1,66 @@ +defmodule CadetWeb.AICommentsHelpers do + @moduledoc """ + Helper functions for Managing LLM related logic + """ + require Logger + + def decrypt_llm_api_key(nil), do: nil + + def decrypt_llm_api_key(encrypted_key) do + case Application.get_env(:openai, :encryption_key) do + secret when is_binary(secret) and byte_size(secret) >= 16 -> + key = binary_part(secret, 0, min(32, byte_size(secret))) + + case String.split(encrypted_key, ":", parts: 3, trim: false) do + [iv_b64, tag_b64, cipher_b64] -> + with {:ok, iv} <- Base.decode64(iv_b64), + {:ok, tag} <- Base.decode64(tag_b64), + {:ok, ciphertext} <- Base.decode64(cipher_b64) do + case :crypto.crypto_one_time_aead(:aes_gcm, key, iv, ciphertext, "", tag, false) do + plain_text when is_binary(plain_text) -> {:ok, plain_text} + _ -> {:decrypt_error, :decryption_failed} + end + else + _ -> + Logger.error("Failed to decode one of the components of the encrypted key") + {:decrypt_error, :invalid_format} + end + + _ -> + Logger.error("Encrypted key format is invalid") + {:decrypt_error, :invalid_format} + end + + _ -> + Logger.error("Encryption key not configured") + {:decrypt_error, :invalid_encryption_key} + end + end + + def encrypt_llm_api_key(llm_api_key) do + secret = Application.get_env(:openai, :encryption_key) + + if is_binary(secret) and byte_size(secret) >= 16 do + # Use first 16 bytes for AES-128, 24 for AES-192, or 32 for AES-256 + key = binary_part(secret, 0, min(32, byte_size(secret))) + # Use AES in GCM mode for encryption + iv = :crypto.strong_rand_bytes(16) + + {ciphertext, tag} = + :crypto.crypto_one_time_aead( + :aes_gcm, + key, + iv, + llm_api_key, + "", + true + ) + + # Store both the IV, ciphertext and tag + encrypted = + Base.encode64(iv) <> ":" <> Base.encode64(tag) <> ":" <> Base.encode64(ciphertext) + else + {:error, :invalid_encryption_key} + end + end +end diff --git a/lib/cadet_web/router.ex b/lib/cadet_web/router.ex index a2b13b72a..8c917c918 100644 --- a/lib/cadet_web/router.ex +++ b/lib/cadet_web/router.ex @@ -220,12 +220,30 @@ defmodule CadetWeb.Router do post("/grading/:submissionid/autograde", AdminGradingController, :autograde_submission) post("/grading/:submissionid/:questionid", AdminGradingController, :update) + post( + "/generate-comments/:answer_id", + AICodeAnalysisController, + :generate_ai_comments + ) + post( "/grading/:submissionid/:questionid/autograde", AdminGradingController, :autograde_answer ) + post( + "/save-final-comment/:answer_id", + AICodeAnalysisController, + :save_final_comment + ) + + post( + "/save-chosen-comments/:submissionid/:questionid", + AICodeAnalysisController, + :save_chosen_comments + ) + get("/users", AdminUserController, :index) get("/users/teamformation", AdminUserController, :get_students) put("/users", AdminUserController, :upsert_users_and_groups) diff --git a/lib/cadet_web/views/courses_view.ex b/lib/cadet_web/views/courses_view.ex index b1db95a07..a3a3f443e 100644 --- a/lib/cadet_web/views/courses_view.ex +++ b/lib/cadet_web/views/courses_view.ex @@ -16,6 +16,10 @@ defmodule CadetWeb.CoursesView do topContestLeaderboardDisplay: :top_contest_leaderboard_display, enableSourcecast: :enable_sourcecast, enableStories: :enable_stories, + enableLlmGrading: :enable_llm_grading, + llmModel: :llm_model, + llmApiUrl: :llm_api_url, + llmCourseLevelPrompt: :llm_course_level_prompt, sourceChapter: :source_chapter, sourceVariant: :source_variant, moduleHelpText: :module_help_text, diff --git a/mix.exs b/mix.exs index c76c48082..8120ffa23 100644 --- a/mix.exs +++ b/mix.exs @@ -67,9 +67,9 @@ defmodule Cadet.Mixfile do {:guardian, "~> 2.0"}, {:guardian_db, "~> 2.0"}, {:hackney, "~> 1.6"}, - {:httpoison, "~> 1.6"}, + {:httpoison, "~> 1.6", override: true}, {:jason, "~> 1.2"}, - {:openai, "~> 0.4.1"}, + {:openai, "~> 0.6.2"}, {:openid_connect, "~> 0.2"}, {:phoenix, "~> 1.5"}, {:phoenix_view, "~> 2.0"}, diff --git a/mix.lock b/mix.lock index 1b5b1828b..8c33b0ab9 100644 --- a/mix.lock +++ b/mix.lock @@ -72,7 +72,7 @@ "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_ownership": {:hex, :nimble_ownership, "1.0.1", "f69fae0cdd451b1614364013544e66e4f5d25f36a2056a9698b793305c5aa3a6", [:mix], [], "hexpm", "3825e461025464f519f3f3e4a1f9b68c47dc151369611629ad08b636b73bb22d"}, "oban": {:hex, :oban, "2.18.0", "092d20bfd3d70c7ecb70960f8548d300b54bb9937c7f2e56b388f3a9ed02ec68", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "aace1eff6f8227ae38d4274af967d96f051c2f0a5152f2ef9809dd1f97866745"}, - "openai": {:hex, :openai, "0.4.1", "c141283d35fed72eaeee22397f0a3eaa93515283b3a698cd3aa0af9301bb5fc5", [:mix], [{:httpoison, "~> 1.8", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "d958993926474362715152d5d4f68444038d53e649641fe1af2fcb26ca8bbe90"}, + "openai": {:hex, :openai, "0.6.2", "48ee0dc74f4d0327ebf78eaeeed8c3595e10eca88fd2a7cdd8b53473645078d6", [:mix], [{:httpoison, "~> 2.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "508c8c2937ef8627d111d9142ff9cc284d39cd0c9b8244339551ac5f5fe0e643"}, "openid_connect": {:hex, :openid_connect, "0.2.2", "c05055363330deab39ffd89e609db6b37752f255a93802006d83b45596189c0b", [:mix], [{:httpoison, "~> 1.2", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "735769b6d592124b58edd0582554ce638524c0214cd783d8903d33357d74cc13"}, "parallel_stream": {:hex, :parallel_stream, "1.1.0", "f52f73eb344bc22de335992377413138405796e0d0ad99d995d9977ac29f1ca9", [:mix], [], "hexpm", "684fd19191aedfaf387bbabbeb8ff3c752f0220c8112eb907d797f4592d6e871"}, "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, diff --git a/priv/repo/migrations/20251008160219_add_llm_grading_access.exs b/priv/repo/migrations/20251008160219_add_llm_grading_access.exs new file mode 100644 index 000000000..f0f485435 --- /dev/null +++ b/priv/repo/migrations/20251008160219_add_llm_grading_access.exs @@ -0,0 +1,15 @@ +defmodule Cadet.Repo.Migrations.AddLlmGradingAccess do + use Ecto.Migration + + def up do + alter table(:courses) do + add(:enable_llm_grading, :boolean, null: true) + end + end + + def down do + alter table(:courses) do + remove(:enable_llm_grading) + end + end +end diff --git a/priv/repo/migrations/20251019160255_add_llm_api_key_to_courses.exs b/priv/repo/migrations/20251019160255_add_llm_api_key_to_courses.exs new file mode 100644 index 000000000..2c6724822 --- /dev/null +++ b/priv/repo/migrations/20251019160255_add_llm_api_key_to_courses.exs @@ -0,0 +1,21 @@ +defmodule Cadet.Repo.Migrations.AddLlmApiKeyToCourses do + use Ecto.Migration + + def up do + alter table(:courses) do + add(:llm_api_key, :text, null: true) + add(:llm_model, :text, null: true) + add(:llm_api_url, :text, null: true) + add(:llm_course_level_prompt, :text, null: true) + end + end + + def down do + alter table(:courses) do + remove(:llm_course_level_prompt) + remove(:llm_api_key) + remove(:llm_model) + remove(:llm_api_url) + end + end +end diff --git a/priv/repo/migrations/20251022103623_create_ai_comments.exs b/priv/repo/migrations/20251022103623_create_ai_comments.exs new file mode 100644 index 000000000..5093d1a43 --- /dev/null +++ b/priv/repo/migrations/20251022103623_create_ai_comments.exs @@ -0,0 +1,15 @@ +defmodule Cadet.Repo.Migrations.CreateAiCommentLogs do + use Ecto.Migration + + def change do + create table(:ai_comment_logs) do + add(:answer_id, references(:answers, on_delete: :delete_all), null: false) + add(:raw_prompt, :text, null: false) + add(:answers_json, :text, null: false) + add(:response, :text) + add(:error, :text) + add(:final_comment, :text) + timestamps() + end + end +end diff --git a/priv/repo/migrations/20251028050808_add_llm_assessment_prompt_assessment.exs b/priv/repo/migrations/20251028050808_add_llm_assessment_prompt_assessment.exs new file mode 100644 index 000000000..9811ed2a7 --- /dev/null +++ b/priv/repo/migrations/20251028050808_add_llm_assessment_prompt_assessment.exs @@ -0,0 +1,9 @@ +defmodule Cadet.Repo.Migrations.AddLlmAssessmentPromptAssessment do + use Ecto.Migration + + def change do + alter table(:assessments) do + add(:llm_assessment_prompt, :text, default: nil) + end + end +end diff --git a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs index 69cef101b..6554b669b 100644 --- a/test/cadet_web/admin_controllers/admin_grading_controller_test.exs +++ b/test/cadet_web/admin_controllers/admin_grading_controller_test.exs @@ -248,6 +248,7 @@ defmodule CadetWeb.AdminGradingControllerTest do conn = get(conn, build_url(course.id, submission.id)) expected = %{ + "enable_llm_grading" => false, "assessment" => %{ "id" => assessment.id, "title" => assessment.title, @@ -266,6 +267,8 @@ defmodule CadetWeb.AdminGradingControllerTest do &case &1.question.type do :programming -> %{ + "id" => &1.id, + "prompts" => [], "question" => %{ "prepend" => &1.question.question.prepend, "postpend" => &1.question.question.postpend, @@ -315,6 +318,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "autogradingStatus" => Atom.to_string(&1.autograding_status), "autogradingResults" => &1.autograding_results }, + "ai_comments" => nil, "solution" => &1.question.question.solution, "grade" => %{ "xp" => &1.xp, @@ -336,6 +340,8 @@ defmodule CadetWeb.AdminGradingControllerTest do :mcq -> %{ + "id" => &1.id, + "prompts" => [], "question" => %{ "type" => "#{&1.question.type}", "blocking" => &1.question.blocking, @@ -365,6 +371,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "autogradingStatus" => Atom.to_string(&1.autograding_status), "autogradingResults" => &1.autograding_results }, + "ai_comments" => nil, "solution" => "", "grade" => %{ "xp" => &1.xp, @@ -386,6 +393,8 @@ defmodule CadetWeb.AdminGradingControllerTest do :voting -> %{ + "id" => &1.id, + "prompts" => [], "question" => %{ "prepend" => &1.question.question.prepend, "solutionTemplate" => &1.question.question.template, @@ -412,6 +421,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "scoreLeaderboard" => [], "popularVoteLeaderboard" => [] }, + "ai_comments" => nil, "grade" => %{ "xp" => &1.xp, "xpAdjustment" => &1.xp_adjustment, @@ -1272,6 +1282,7 @@ defmodule CadetWeb.AdminGradingControllerTest do conn = get(conn, build_url(course.id, submission.id)) expected = %{ + "enable_llm_grading" => false, "assessment" => %{ "id" => assessment.id, "title" => assessment.title, @@ -1290,6 +1301,8 @@ defmodule CadetWeb.AdminGradingControllerTest do &case &1.question.type do :programming -> %{ + "id" => &1.id, + "prompts" => [], "question" => %{ "prepend" => &1.question.question.prepend, "postpend" => &1.question.question.postpend, @@ -1339,6 +1352,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "autogradingStatus" => Atom.to_string(&1.autograding_status), "autogradingResults" => &1.autograding_results }, + "ai_comments" => nil, "solution" => &1.question.question.solution, "grade" => %{ "xp" => &1.xp, @@ -1360,6 +1374,8 @@ defmodule CadetWeb.AdminGradingControllerTest do :mcq -> %{ + "id" => &1.id, + "prompts" => [], "question" => %{ "type" => "#{&1.question.type}", "blocking" => &1.question.blocking, @@ -1389,6 +1405,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "autogradingStatus" => Atom.to_string(&1.autograding_status), "autogradingResults" => &1.autograding_results }, + "ai_comments" => nil, "solution" => "", "grade" => %{ "xp" => &1.xp, @@ -1410,6 +1427,8 @@ defmodule CadetWeb.AdminGradingControllerTest do :voting -> %{ + "id" => &1.id, + "prompts" => [], "question" => %{ "prepend" => &1.question.question.prepend, "solutionTemplate" => &1.question.question.template, @@ -1436,6 +1455,7 @@ defmodule CadetWeb.AdminGradingControllerTest do "scoreLeaderboard" => [], "popularVoteLeaderboard" => [] }, + "ai_comments" => nil, "grade" => %{ "xp" => &1.xp, "xpAdjustment" => &1.xp_adjustment, diff --git a/test/cadet_web/controllers/ai_code_analysis_controller_test.exs b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs new file mode 100644 index 000000000..c6e1ca3a4 --- /dev/null +++ b/test/cadet_web/controllers/ai_code_analysis_controller_test.exs @@ -0,0 +1,143 @@ +import Mock + +defmodule CadetWeb.AICodeAnalysisControllerTest do + use CadetWeb.ConnCase + alias Cadet.{Repo, AIComments} + alias Cadet.{AIComments.AIComment, Courses.Course} + alias CadetWeb.AICommentsHelpers + + setup do + course_with_llm = + insert(:course, %{ + enable_llm_grading: true, + llm_api_key: AICommentsHelpers.encrypt_llm_api_key("test_key"), + llm_model: "gpt-5-mini", + llm_api_url: "http://testapi.com", + llm_course_level_prompt: "Example Prompt" + }) + + example_assessment = insert(:assessment, %{course: course_with_llm}) + new_submission = insert(:submission, %{assessment: example_assessment}) + question = insert(:programming_question, %{assessment: example_assessment}) + answer = insert(:answer, %{submission: new_submission, question: question}) + admin_user = insert(:course_registration, %{role: :admin, course: course_with_llm}) + staff_user = insert(:course_registration, %{role: :staff, course: course_with_llm}) + + {:ok, + %{ + admin_user: admin_user, + staff_user: staff_user, + course_with_llm: course_with_llm, + example_assessment: example_assessment, + new_submission: new_submission, + question: question, + answer: answer + }} + end + + describe "GET /v2/courses/:course_id/admin/generate-comments/:answer_id" do + test "success with happy path, admin and staff", %{ + conn: conn, + admin_user: admin_user, + staff_user: staff_user, + course_with_llm: course_with_llm, + example_assessment: example_assessment, + new_submission: new_submission, + question: question, + answer: answer + } do + # Make the API call + with_mock OpenAI, + chat_completion: fn _input, _overrides -> + {:ok, %{:choices => [%{"message" => %{"content" => "Comment1|||Comment2"}}]}} + end do + conn + |> sign_in(staff_user.user) + |> post(build_url_generate_ai_comments(course_with_llm.id, answer.id)) + |> json_response(200) + end + + with_mock OpenAI, + chat_completion: fn _input, _overrides -> + {:ok, %{:choices => [%{"message" => %{"content" => "Comment1|||Comment2"}}]}} + end do + response = + conn + |> sign_in(admin_user.user) + |> post(build_url_generate_ai_comments(course_with_llm.id, answer.id)) + |> json_response(200) + + # Verify response + assert response["comments"] == ["Comment1", "Comment2"] + end + + # Verify database entry + comments = Repo.all(AIComment) + assert length(comments) > 0 + latest_comment = List.first(comments) + assert latest_comment.answer_id == answer.id + assert latest_comment.raw_prompt != nil + assert latest_comment.answers_json != nil + end + + test "errors out when given an invalid answer id", %{ + conn: conn, + admin_user: admin_user, + staff_user: staff_user, + course_with_llm: course_with_llm, + example_assessment: example_assessment, + new_submission: new_submission, + question: question, + answer: answer + } do + random_answer_id = 324_324 + + # Make the API call that should fail + with_mock OpenAI, [:passthrough], + chat_completion: fn _input, _overrides -> + {:ok, %{:choices => [%{"message" => %{"content" => "Comment1|||Comment2"}}]}} + end do + response = + conn + |> sign_in(admin_user.user) + |> post(build_url_generate_ai_comments(course_with_llm.id, random_answer_id)) + |> text_response(400) + end + end + + test "LLM endpoint returns an invalid response - should log errors in database", %{ + conn: conn, + admin_user: admin_user, + staff_user: staff_user, + course_with_llm: course_with_llm, + example_assessment: example_assessment, + new_submission: new_submission, + question: question, + answer: answer + } do + # Make the API call that should fail + with_mock OpenAI, [:passthrough], + chat_completion: fn _input, _overrides -> + {:ok, %{"body" => "Some unexpected response"}} + end do + response = + conn + |> sign_in(admin_user.user) + |> post(build_url_generate_ai_comments(course_with_llm.id, answer.id)) + |> text_response(502) + end + + # Verify database entry even with error + comments = Repo.all(AIComment) + assert length(comments) > 0 + latest_comment = List.first(comments) + assert latest_comment.answer_id == answer.id + assert latest_comment.raw_prompt != nil + assert latest_comment.answers_json != nil + end + end + + defp build_url_generate_ai_comments(course_id, answer_id) do + "/v2/courses/#{course_id}/admin/generate-comments/#{answer_id}" + end +end diff --git a/test/factories/assessments/assessment_factory.ex b/test/factories/assessments/assessment_factory.ex index e85267418..5dba2955d 100644 --- a/test/factories/assessments/assessment_factory.ex +++ b/test/factories/assessments/assessment_factory.ex @@ -39,7 +39,8 @@ defmodule Cadet.Assessments.AssessmentFactory do open_at: Timex.now(), close_at: Timex.shift(Timex.now(), days: Enum.random(1..30)), is_published: false, - max_team_size: 1 + max_team_size: 1, + llm_assessment_prompt: nil } end end diff --git a/test/factories/assessments/question_factory.ex b/test/factories/assessments/question_factory.ex index 783db1180..296924dc3 100644 --- a/test/factories/assessments/question_factory.ex +++ b/test/factories/assessments/question_factory.ex @@ -51,7 +51,8 @@ defmodule Cadet.Assessments.QuestionFactory do answer: Faker.StarWars.character(), program: Faker.Lorem.Shakespeare.king_richard_iii() } - ] + ], + llm_prompt: nil } end diff --git a/test/factories/courses/course_factory.ex b/test/factories/courses/course_factory.ex index 0ca2c3ec6..e9d229803 100644 --- a/test/factories/courses/course_factory.ex +++ b/test/factories/courses/course_factory.ex @@ -18,7 +18,8 @@ defmodule Cadet.Courses.CourseFactory do enable_stories: false, source_chapter: 1, source_variant: "default", - module_help_text: "Help Text" + module_help_text: "Help Text", + enable_llm_grading: false } end end diff --git a/test/support/seeds.ex b/test/support/seeds.ex index e263860e1..1f0b998ac 100644 --- a/test/support/seeds.ex +++ b/test/support/seeds.ex @@ -236,7 +236,8 @@ defmodule Cadet.Test.Seeds do display_order: id, assessment: assessment, max_xp: 1000, - show_solution: assessment.config.type == "path" + show_solution: assessment.config.type == "path", + question: build(:programming_question_content) }) end)