From f5105967dfe22c6cdafcbea454b8d7458288d8cd Mon Sep 17 00:00:00 2001 From: leon Date: Sat, 16 Nov 2024 19:30:21 +0100 Subject: [PATCH 01/18] Refactor feedback reference --- athena/athena/models/db_modeling_feedback.py | 4 +-- athena/athena/schemas/modeling_feedback.py | 6 ++-- .../apollon_json_transformer.py | 8 +++-- .../apollon_transformer/parser/uml_parser.py | 10 ++++++- .../models/assessment_model.py | 2 +- .../models/exercise_model.py | 3 +- .../utils/convert_to_athana_feedback_model.py | 29 ++++++++++++------- .../utils/get_exercise_model.py | 5 ++-- 8 files changed, 42 insertions(+), 25 deletions(-) diff --git a/athena/athena/models/db_modeling_feedback.py b/athena/athena/models/db_modeling_feedback.py index 142a59911..1fdffd23c 100644 --- a/athena/athena/models/db_modeling_feedback.py +++ b/athena/athena/models/db_modeling_feedback.py @@ -1,6 +1,6 @@ from typing import Optional -from sqlalchemy import Column, ForeignKey, JSON +from sqlalchemy import Column, ForeignKey, String from sqlalchemy.orm import relationship from athena.database import Base @@ -11,7 +11,7 @@ class DBModelingFeedback(DBFeedback, Base): __tablename__ = "modeling_feedbacks" - element_ids: Optional[list[str]] = Column(JSON) # type: ignore + reference: Optional[str] = Column(String, nullable=True) # type: ignore exercise_id = Column(BigIntegerWithAutoincrement, ForeignKey("modeling_exercises.id", ondelete="CASCADE"), index=True) submission_id = Column(BigIntegerWithAutoincrement, ForeignKey("modeling_submissions.id", ondelete="CASCADE"), index=True) diff --git a/athena/athena/schemas/modeling_feedback.py b/athena/athena/schemas/modeling_feedback.py index f77322f43..85245c2e0 100644 --- a/athena/athena/schemas/modeling_feedback.py +++ b/athena/athena/schemas/modeling_feedback.py @@ -1,11 +1,9 @@ -from typing import Optional, List - +from typing import Optional from pydantic import Field - from .feedback import Feedback class ModelingFeedback(Feedback): """Feedback on a modeling exercise.""" - element_ids: Optional[List[str]] = Field([], description="referenced diagram element IDs", example=["id_1"]) + reference: Optional[str] = Field(None, description="reference to the diagram element", example="ClassAttribute:5a337bdf-da00-4bd0-a6f0-78ba5b84330e") \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py index 3d67196df..7a487bfff 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py @@ -6,7 +6,7 @@ class ApollonJSONTransformer: @staticmethod - def transform_json(model: str) -> tuple[str, dict[str, str], str]: + def transform_json(model: str) -> tuple[str, dict[str, str], str, dict[str, str]]: """ Serialize a given Apollon diagram model to a string representation. This method converts the UML diagram model into a format similar to mermaid syntax, called "apollon". @@ -30,6 +30,8 @@ def transform_json(model: str) -> tuple[str, dict[str, str], str]: **{element['name']: element['id'] for element in parser.get_elements()}, **{relation['name']: relation['id'] for relation in parser.get_relations()} } - - return apollon_representation, names, diagram_type + + id_type_mapping = parser.get_id_to_type_mapping() + + return apollon_representation, names, diagram_type, id_type_mapping \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py index 57b9a63ad..97d7c43bb 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py @@ -93,4 +93,12 @@ def get_elements(self) -> List[Element]: return self.elements def get_relations(self) -> List[Relation]: - return self.relations \ No newline at end of file + return self.relations + + def get_id_to_type_mapping(self) -> Dict[str, str]: + id_to_type = {} + for element in self.elements: + id_to_type[element.id] = element.type + for relation in self.relations: + id_to_type[relation.id] = relation.type + return id_to_type \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py index 9ccc1933d..3468f7942 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py @@ -4,7 +4,7 @@ class FeedbackModel(BaseModel): title: str = Field(description="Very short title, i.e. feedback category or similar", example="Logic Error") description: str = Field(description="Feedback description") - element_names: Optional[List[str]] = Field(description="Referenced diagram element names, and relations (R) or empty if unreferenced") + element_name: Optional[str] = Field(description="Referenced diagram element, attribute names, and relations (R) or empty if unreferenced") credits: float = Field(0.0, description="Number of points received/deducted") grading_instruction_id: int = Field( description="ID of the grading instruction that was used to generate this feedback" diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py index 3a6f0672f..78a07a19c 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py @@ -11,4 +11,5 @@ class ExerciseModel(BaseModel): grading_instructions: Optional[str] = None submission_uml_type: str transformed_example_solution: Optional[str] = None - element_id_mapping: dict[str, str] \ No newline at end of file + element_id_mapping: dict[str, str] + id_type_mapping: dict[str, str] \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py index 06b4b5a66..dd44dfa71 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py @@ -11,33 +11,40 @@ def convert_to_athana_feedback_model( manual_structured_grading_instructions: Optional[List[GradingCriterion]] = None ) -> List[Feedback]: - grading_instruction_ids = set( + grading_instruction_ids = { grading_instruction.id - for criterion in manual_structured_grading_instructions or [] + for criterion in (manual_structured_grading_instructions or []) for grading_instruction in criterion.structured_grading_instructions - ) + } feedbacks = [] for feedback in feedback_result.feedbacks: # Each feedback has a grading_instruction_id. However we only want to have the grading_instruction_id in the final feedback that are associated with the manual structured grading instructions - grading_instruction_id = feedback.grading_instruction_id if feedback.grading_instruction_id in grading_instruction_ids else None - element_ids = [ - exercise_model.element_id_mapping[element] - for element in (feedback.element_names or []) - if element in exercise_model.element_id_mapping - ] + grading_instruction_id = ( + feedback.grading_instruction_id + if feedback.grading_instruction_id in grading_instruction_ids + else None + ) + + reference: Optional[str] = None + if feedback.element_name: + reference_id = exercise_model.element_id_mapping.get(feedback.element_name) + reference_type = exercise_model.id_type_mapping.get(reference_id) if reference_id else None + + if reference_type and reference_id: + reference = f"{reference_type}:{reference_id}" feedbacks.append(Feedback( exercise_id=exercise_model.exercise_id, submission_id=exercise_model.submission_id, title=feedback.title, description=feedback.description, - element_ids=element_ids, credits=feedback.credits, structured_grading_instruction_id=grading_instruction_id, meta={}, id=None, - is_graded=False + is_graded=False, + reference=reference )) return feedbacks \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/get_exercise_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/get_exercise_model.py index a2457f1f4..3489be738 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/get_exercise_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/get_exercise_model.py @@ -7,9 +7,9 @@ def get_exercise_model(exercise: Exercise, submission: Submission) -> ExerciseMo serialized_example_solution = None if exercise.example_solution: - serialized_example_solution, _, _ = ApollonJSONTransformer.transform_json(exercise.example_solution) + serialized_example_solution, _, _, _ = ApollonJSONTransformer.transform_json(exercise.example_solution) - transformed_submission, element_id_mapping, diagram_type = ApollonJSONTransformer.transform_json(submission.model) + transformed_submission, element_id_mapping, diagram_type, id_type_mapping = ApollonJSONTransformer.transform_json(submission.model) return ExerciseModel( submission_id=submission.id, @@ -22,6 +22,7 @@ def get_exercise_model(exercise: Exercise, submission: Submission) -> ExerciseMo submission_uml_type=diagram_type, transformed_example_solution=serialized_example_solution, element_id_mapping=element_id_mapping, + id_type_mapping=id_type_mapping ) From 6b98bb862aa3b2cdf4db57ffcb70c9578e2cbf2d Mon Sep 17 00:00:00 2001 From: leon Date: Mon, 18 Nov 2024 08:36:45 +0100 Subject: [PATCH 02/18] Enhance Apollon JSON transformer and parser for improved element ID mapping and uniqueness resolution --- .../apollon_json_transformer.py | 8 +-- .../apollon_transformer/parser/element.py | 61 +++++++++++++++++-- .../apollon_transformer/parser/uml_parser.py | 30 +++++++-- .../models/assessment_model.py | 4 +- .../models/exercise_model.py | 6 +- .../prompts/graded_feedback_prompt.py | 12 +++- .../utils/convert_to_athana_feedback_model.py | 9 ++- 7 files changed, 100 insertions(+), 30 deletions(-) diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py index 7a487bfff..2142a2cdb 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/apollon_json_transformer.py @@ -25,11 +25,9 @@ def transform_json(model: str) -> tuple[str, dict[str, str], str, dict[str, str] # Convert the UML diagram to the apollon representation apollon_representation = parser.to_apollon() - # Extract elements and relations with their corresponding IDs and names - names = { - **{element['name']: element['id'] for element in parser.get_elements()}, - **{relation['name']: relation['id'] for relation in parser.get_relations()} - } + # Get the mapping of element, method, and attribute names to their corresponding IDs + # This is used to resolve references to as the apollon representation only contains names and not IDs + names = parser.get_element_id_mapping() id_type_mapping = parser.get_id_to_type_mapping() diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/element.py b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/element.py index 415b586a6..ef9c52a79 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/element.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/element.py @@ -1,4 +1,7 @@ -from typing import Dict, Any, List, Optional +# apollon_transformer/parser/element.py + +from typing import Dict, Any, List, Optional, Tuple +from string import ascii_uppercase class Element: """ @@ -17,15 +20,61 @@ def __init__(self, data: Dict[str, Any], element_dict: Optional[Dict[str, Any]] self.method_refs: List[str] = data.get('methods', []) self.attributes: List[str] = [] self.methods: List[str] = [] + self.attribute_id_mapping: Dict[str, str] = {} + self.method_id_mapping: Dict[str, str] = {} if element_dict is not None: self.resolve_references(element_dict) def resolve_references(self, element_dict: Dict[str, Any]): """ - Resolve attribute and method references using the provided element dictionary. The json data contains only references to other elements that represent attributes and methods. This method resolves these references to the actual names of the attributes and methods by looking up the corresponding elements via their IDs in the provided element dictionary. + Resolve attribute and method references using the provided element dictionary. + Ensures uniqueness among attribute and method names within the class. """ - self.attributes = [element_dict[ref].get("name", "") for ref in self.attribute_refs if ref in element_dict] - self.methods = [element_dict[ref].get('name', '') for ref in self.method_refs if ref in element_dict] + # Resolve attributes + self.attributes, self.attribute_id_mapping = self._resolve_uniqueness( + self.attribute_refs, element_dict) + + # Resolve methods + self.methods, self.method_id_mapping = self._resolve_uniqueness( + self.method_refs, element_dict) + + def _resolve_uniqueness( + self, refs: List[str], element_dict: Dict[str, Any] + ) -> Tuple[List[str], Dict[str, str]]: + name_counts: Dict[str, int] = {} + unique_full_names: List[str] = [] + id_mapping: Dict[str, str] = {} + for ref in refs: + if ref in element_dict: + full_name = element_dict[ref].get("name", "") + simplified_name = self.extract_name(full_name) + count = name_counts.get(simplified_name, 0) + if count > 0: + suffix = f"#{ascii_uppercase[count - 1]}" + simplified_name_with_suffix = f"{simplified_name}{suffix}" + else: + simplified_name_with_suffix = simplified_name + name_counts[simplified_name] = count + 1 + unique_full_names.append(full_name) + id_mapping[simplified_name_with_suffix] = ref + return unique_full_names, id_mapping + + @staticmethod + def extract_name(full_name: str) -> str: + """ + Extracts the simplified name from the full attribute or method name. + Removes visibility symbols, type annotations, and parameters. + """ + # Remove visibility symbols and leading/trailing spaces + name = full_name.lstrip('+-#~ ').strip() + # For attributes, split on ':' + if ':' in name: + name = name.split(':')[0].strip() + # For methods, split on '(' + elif '(' in name: + name = name.split('(')[0].strip() + return name + def to_apollon(self) -> str: parts = [f"[{self.type}] {self.name}"] @@ -41,6 +90,6 @@ def to_apollon(self) -> str: parts.append("{\n" + "\n".join(details) + "\n}") return " ".join(parts) - + def __getitem__(self, key): - return self.__dict__[key] \ No newline at end of file + return self.__dict__[key] diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py index 97d7c43bb..04b6ae6bc 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/apollon_transformer/parser/uml_parser.py @@ -94,11 +94,29 @@ def get_elements(self) -> List[Element]: def get_relations(self) -> List[Relation]: return self.relations - - def get_id_to_type_mapping(self) -> Dict[str, str]: - id_to_type = {} + + def get_element_id_mapping(self) -> Dict[str, str]: + """ + Creates a mapping from element names to their IDs, including attributes and methods. + """ + mapping = {} for element in self.elements: - id_to_type[element.id] = element.type + mapping[element.name] = element.id + for simplified_name_with_suffix, attr_id in element.attribute_id_mapping.items(): + mapping[f"{element.name}.{simplified_name_with_suffix}"] = attr_id + for simplified_name_with_suffix, method_id in element.method_id_mapping.items(): + mapping[f"{element.name}.{simplified_name_with_suffix}"] = method_id for relation in self.relations: - id_to_type[relation.id] = relation.type - return id_to_type \ No newline at end of file + mapping[relation.name] = relation.id + return mapping + + def get_id_to_type_mapping(self) -> Dict[str, str]: + """ + Creates a mapping from IDs to their types, including attributes and methods. + """ + mapping = {} + for element in self.data['elements'].values(): + mapping[element['id']] = element['type'] + for relation in self.data['relationships'].values(): + mapping[relation['id']] = relation['type'] + return mapping \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py index 3468f7942..29ccdb55a 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/models/assessment_model.py @@ -1,10 +1,10 @@ -from typing import List, Optional, Sequence +from typing import Optional, Sequence from pydantic import BaseModel, Field class FeedbackModel(BaseModel): title: str = Field(description="Very short title, i.e. feedback category or similar", example="Logic Error") description: str = Field(description="Feedback description") - element_name: Optional[str] = Field(description="Referenced diagram element, attribute names, and relations (R) or empty if unreferenced") + element_name: Optional[str] = Field(description="Referenced diagram element, attribute names, and relations (use format: , ., ., R), or leave empty if unreferenced") credits: float = Field(0.0, description="Number of points received/deducted") grading_instruction_id: int = Field( description="ID of the grading instruction that was used to generate this feedback" diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py index 78a07a19c..1d9eef7c3 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/models/exercise_model.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, Dict from pydantic import BaseModel class ExerciseModel(BaseModel): @@ -11,5 +11,5 @@ class ExerciseModel(BaseModel): grading_instructions: Optional[str] = None submission_uml_type: str transformed_example_solution: Optional[str] = None - element_id_mapping: dict[str, str] - id_type_mapping: dict[str, str] \ No newline at end of file + element_id_mapping: Dict[str, str] + id_type_mapping: Dict[str, str] \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/graded_feedback_prompt.py b/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/graded_feedback_prompt.py index f2fa1390f..71662c84b 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/graded_feedback_prompt.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/graded_feedback_prompt.py @@ -42,9 +42,15 @@ class GradedFeedbackInputs(BaseModel): {example_solution} Important: -Make sure to provide detailed feedback for each criterion. Always try to be as specific as possible. -Also make sure your feedback adds up to the correct number of points. If there are n points available and everything is correct, then the feedback should add up to n points. -Deeply think about the diagram and what the student potentially missed, misunderstood or mixed up. +- Make sure to provide detailed feedback for each criterion. Always try to be as specific as possible. +- Also make sure your feedback adds up to the correct number of points. If there are n points available and everything is correct, then the feedback should add up to n points. +- Deeply think about the diagram and what the student potentially missed, misunderstood, or mixed up. +- For the `element_name` field in the output, reference the specific diagram element, attribute, method, or relation related to the feedback. Use the following formats: + - For classes or elements: `` + - For attributes: `.` + - For methods: `.` + - For relations: `R` (e.g., `R1`, `R2`) +- If the feedback is not related to a specific element, leave the `element_name` field empty. The submission uses the following UML diagram format: diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py index dd44dfa71..fd96d3bbf 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py @@ -6,8 +6,8 @@ def convert_to_athana_feedback_model( - feedback_result : AssessmentModel, - exercise_model: ExerciseModel, + feedback_result: AssessmentModel, + exercise_model: ExerciseModel, manual_structured_grading_instructions: Optional[List[GradingCriterion]] = None ) -> List[Feedback]: @@ -19,10 +19,9 @@ def convert_to_athana_feedback_model( feedbacks = [] for feedback in feedback_result.feedbacks: - # Each feedback has a grading_instruction_id. However we only want to have the grading_instruction_id in the final feedback that are associated with the manual structured grading instructions grading_instruction_id = ( - feedback.grading_instruction_id - if feedback.grading_instruction_id in grading_instruction_ids + feedback.grading_instruction_id + if feedback.grading_instruction_id in grading_instruction_ids else None ) From 9de3f10f87b755b50b188b1826339770f9a09af0 Mon Sep 17 00:00:00 2001 From: leon Date: Wed, 27 Nov 2024 19:07:42 +0100 Subject: [PATCH 03/18] Add element_ids field to ModelingFeedback and update feedback conversion logic --- athena/athena/schemas/modeling_feedback.py | 3 ++- .../utils/convert_to_athana_feedback_model.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/athena/athena/schemas/modeling_feedback.py b/athena/athena/schemas/modeling_feedback.py index 85245c2e0..8b5123536 100644 --- a/athena/athena/schemas/modeling_feedback.py +++ b/athena/athena/schemas/modeling_feedback.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import List, Optional from pydantic import Field from .feedback import Feedback @@ -6,4 +6,5 @@ class ModelingFeedback(Feedback): """Feedback on a modeling exercise.""" + element_ids: Optional[List[str]] = Field([], description="referenced diagram element IDs", example=["id_1"]) reference: Optional[str] = Field(None, description="reference to the diagram element", example="ClassAttribute:5a337bdf-da00-4bd0-a6f0-78ba5b84330e") \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py index fd96d3bbf..0a6b260c5 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py @@ -43,7 +43,8 @@ def convert_to_athana_feedback_model( meta={}, id=None, is_graded=False, - reference=reference + reference=reference, + element_ids=[reference_id] if reference_id else [] )) return feedbacks \ No newline at end of file From 1e8e21da11fad847956dd3f6b5f9b9c7d6849bf1 Mon Sep 17 00:00:00 2001 From: leon Date: Thu, 28 Nov 2024 12:22:48 +0100 Subject: [PATCH 04/18] Add element_ids field to DBModelingFeedback model --- athena/athena/models/db_modeling_feedback.py | 1 + 1 file changed, 1 insertion(+) diff --git a/athena/athena/models/db_modeling_feedback.py b/athena/athena/models/db_modeling_feedback.py index 1fdffd23c..d121cfe98 100644 --- a/athena/athena/models/db_modeling_feedback.py +++ b/athena/athena/models/db_modeling_feedback.py @@ -11,6 +11,7 @@ class DBModelingFeedback(DBFeedback, Base): __tablename__ = "modeling_feedbacks" + element_ids: Optional[list[str]] = Column(JSON) # type: ignore reference: Optional[str] = Column(String, nullable=True) # type: ignore exercise_id = Column(BigIntegerWithAutoincrement, ForeignKey("modeling_exercises.id", ondelete="CASCADE"), index=True) From a5203ff5c528b44f1202c03e6cfd016d5a39543d Mon Sep 17 00:00:00 2001 From: leon Date: Fri, 29 Nov 2024 18:28:04 +0100 Subject: [PATCH 05/18] Add JSON type import to db_modeling_feedback.py --- athena/athena/models/db_modeling_feedback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/athena/athena/models/db_modeling_feedback.py b/athena/athena/models/db_modeling_feedback.py index d121cfe98..47d3a636e 100644 --- a/athena/athena/models/db_modeling_feedback.py +++ b/athena/athena/models/db_modeling_feedback.py @@ -1,6 +1,6 @@ from typing import Optional -from sqlalchemy import Column, ForeignKey, String +from sqlalchemy import Column, ForeignKey, JSON, String from sqlalchemy.orm import relationship from athena.database import Base From 7b488a36cd0952e42f21514543e71c516f7ea6b6 Mon Sep 17 00:00:00 2001 From: leon Date: Tue, 3 Dec 2024 13:26:49 +0100 Subject: [PATCH 06/18] add structured grading instruction cache --- athena/athena/models/__init__.py | 3 +- .../models/db_structured_grading_criterion.py | 17 ++++++++++ athena/athena/schemas/__init__.py | 3 +- athena/athena/schemas/grading_criterion.py | 5 +-- .../schemas/structured_grading_criterion.py | 7 ++++ .../structured_grading_criterion_storage.py | 32 ++++++++++++++++++ .../core/generate_suggestions.py | 2 +- .../get_structured_grading_instructions.py | 33 +++++++++++++++++-- 8 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 athena/athena/models/db_structured_grading_criterion.py create mode 100644 athena/athena/schemas/structured_grading_criterion.py create mode 100644 athena/athena/storage/structured_grading_criterion_storage.py diff --git a/athena/athena/models/__init__.py b/athena/athena/models/__init__.py index 5febe42a6..8f63cf88c 100644 --- a/athena/athena/models/__init__.py +++ b/athena/athena/models/__init__.py @@ -14,4 +14,5 @@ from .db_modeling_submission import DBModelingSubmission from .db_programming_feedback import DBProgrammingFeedback from .db_text_feedback import DBTextFeedback -from .db_modeling_feedback import DBModelingFeedback \ No newline at end of file +from .db_modeling_feedback import DBModelingFeedback +from .db_structured_grading_criterion import DBStructuredGradingCriterion \ No newline at end of file diff --git a/athena/athena/models/db_structured_grading_criterion.py b/athena/athena/models/db_structured_grading_criterion.py new file mode 100644 index 000000000..897e54c69 --- /dev/null +++ b/athena/athena/models/db_structured_grading_criterion.py @@ -0,0 +1,17 @@ +from sqlalchemy import Column, JSON, String, ForeignKey +from sqlalchemy.orm import relationship + +from athena.database import Base +from .big_integer_with_autoincrement import BigIntegerWithAutoincrement + + +class DBStructuredGradingCriterion(Base): + __tablename__ = "structured_grading_criterion" + id = Column(BigIntegerWithAutoincrement, primary_key=True, index=True, + autoincrement=True) + exercise_id = Column(BigIntegerWithAutoincrement, ForeignKey("exercises.id", ondelete="CASCADE"), index=True, unique=True) # Only one cached instruction per exercise + instructions_hash = Column(String, nullable=False) + structured_grading_criterion = Column(JSON, nullable=False) + lms_url = Column(String, nullable=False) + + exercise = relationship("DBExercise", back_populates="structured_grading_criterion") \ No newline at end of file diff --git a/athena/athena/schemas/__init__.py b/athena/athena/schemas/__init__.py index 244c2d9c0..b5f6a7943 100644 --- a/athena/athena/schemas/__init__.py +++ b/athena/athena/schemas/__init__.py @@ -13,4 +13,5 @@ from .modeling_feedback import ModelingFeedback from .modeling_exercise import ModelingExercise from .modeling_submission import ModelingSubmission -from .grading_criterion import GradingCriterion, StructuredGradingInstruction, StructuredGradingCriterion \ No newline at end of file +from .grading_criterion import GradingCriterion, StructuredGradingInstruction +from .structured_grading_criterion import StructuredGradingCriterion \ No newline at end of file diff --git a/athena/athena/schemas/grading_criterion.py b/athena/athena/schemas/grading_criterion.py index 15225a7dc..186478830 100644 --- a/athena/athena/schemas/grading_criterion.py +++ b/athena/athena/schemas/grading_criterion.py @@ -23,7 +23,4 @@ class GradingCriterion(Schema, ABC): title: Optional[str] = Field(None, example="Some instructions") structured_grading_instructions: List[StructuredGradingInstruction] = Field( [], example=[{"credits": 1.0, "gradingScale": "Good", "instructionDescription": "Some instructions", "feedback": "Nicely done!", "usageCount": 1}, - {"credits": 0.0, "gradingScale": "Bad", "instructionDescription": "Some instructions", "feedback": "Try again!", "usageCount": 0}]) - -class StructuredGradingCriterion(BaseModel): - criteria: List[GradingCriterion] \ No newline at end of file + {"credits": 0.0, "gradingScale": "Bad", "instructionDescription": "Some instructions", "feedback": "Try again!", "usageCount": 0}]) \ No newline at end of file diff --git a/athena/athena/schemas/structured_grading_criterion.py b/athena/athena/schemas/structured_grading_criterion.py new file mode 100644 index 000000000..89f1beb7b --- /dev/null +++ b/athena/athena/schemas/structured_grading_criterion.py @@ -0,0 +1,7 @@ +from typing import List +from athena.schemas.grading_criterion import GradingCriterion +from pydantic import BaseModel + + +class StructuredGradingCriterion(BaseModel): + criteria: List[GradingCriterion] \ No newline at end of file diff --git a/athena/athena/storage/structured_grading_criterion_storage.py b/athena/athena/storage/structured_grading_criterion_storage.py new file mode 100644 index 000000000..0abd1b3f2 --- /dev/null +++ b/athena/athena/storage/structured_grading_criterion_storage.py @@ -0,0 +1,32 @@ +from typing import Optional +from athena.contextvars import get_lms_url +from athena.database import get_db + +from athena.models import DBStructuredGradingCriterion +from athena.schemas import StructuredGradingCriterion + +def get_structured_grading_criterion(exercise_id: int, current_hash: Optional[str] = None) -> Optional[StructuredGradingCriterion]: + lms_url = get_lms_url() + with get_db() as db: + cache_entry = db.query(DBStructuredGradingCriterion).filter_by( + exercise_id=exercise_id, lms_url=lms_url + ).first() + if cache_entry is not None and (current_hash is None or cache_entry.instructions_hash.is_(current_hash)): + return StructuredGradingCriterion.parse_raw(cache_entry.cached_instructions) + + return None + +def store_structured_grading_criterion( + exercise_id: int, hash: str, structured_instructions: StructuredGradingCriterion +): + lms_url = get_lms_url() + with get_db() as db: + db.merge( + DBStructuredGradingCriterion( + exercise_id=exercise_id, + lms_url=lms_url, + instructions_hash=hash, + cached_instructions=structured_instructions.json(), + ) + ) + db.commit() \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py b/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py index 6db69e7d2..61f051348 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py @@ -1,4 +1,4 @@ -from athena.schemas.grading_criterion import StructuredGradingCriterion +from athena.schemas.structured_grading_criterion import StructuredGradingCriterion from langchain_core.output_parsers import PydanticOutputParser from langchain_core.prompts import ChatPromptTemplate diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py b/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py index ae84e0ddf..4796ae01a 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py @@ -1,9 +1,13 @@ -from typing import List, Optional +import hashlib +import json +from typing import Any, Dict, List, Optional +from athena import logger from athena.metadata import emit_meta +from athena.storage.structured_grading_criterion_storage import get_structured_grading_criterion, store_structured_grading_criterion from langchain_core.output_parsers import PydanticOutputParser from langchain_core.prompts import ChatPromptTemplate -from athena.schemas.grading_criterion import GradingCriterion, StructuredGradingCriterion +from athena.schemas import GradingCriterion, StructuredGradingCriterion from llm_core.utils.predict_and_parse import predict_and_parse from module_modeling_llm.config import BasicApproachConfig from module_modeling_llm.models.exercise_model import ExerciseModel @@ -19,6 +23,12 @@ async def get_structured_grading_instructions( if grading_criteria: return StructuredGradingCriterion(criteria=grading_criteria) + + # Check if we have cached instructions for this exercise + current_hash = get_grading_instructions_hash(exercise_model) + cached_instructions = get_structured_grading_criterion(exercise_model.exercise_id, current_hash) + if cached_instructions: + return cached_instructions chat_prompt = ChatPromptTemplate.from_messages([ ("system", config.generate_suggestions_prompt.structured_grading_instructions_system_message), @@ -53,5 +63,22 @@ async def get_structured_grading_instructions( if not grading_instruction_result: raise ValueError("No structured grading instructions were returned by the model.") + + # Cache the grading instructions + hash = get_grading_instructions_hash(exercise_model) + store_structured_grading_criterion(exercise_model.exercise_id, hash, grading_instruction_result) + + return grading_instruction_result + +def get_grading_instructions_hash(exercise: ExerciseModel) -> str: + + hashable_data = { + "problem_statement": exercise.problem_statement, + "grading_instructions": exercise.grading_instructions, + "sample_solution": exercise.transformed_example_solution, + "max_points": exercise.max_points, + "bonus_points": exercise.bonus_points, + } - return grading_instruction_result \ No newline at end of file + json_string = json.dumps(hashable_data, sort_keys=True, default=str) + return hashlib.sha256(json_string.encode()).hexdigest() \ No newline at end of file From f2604c9b12e0ac3264c4c6276ce9166ad5cf85ad Mon Sep 17 00:00:00 2001 From: leon Date: Tue, 3 Dec 2024 17:14:36 +0100 Subject: [PATCH 07/18] Increase default max_tokens to 4000 in OpenAI model configuration --- llm_core/llm_core/models/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_core/llm_core/models/openai.py b/llm_core/llm_core/models/openai.py index d12df22d4..7bcc0f11f 100644 --- a/llm_core/llm_core/models/openai.py +++ b/llm_core/llm_core/models/openai.py @@ -67,7 +67,7 @@ class OpenAIModelConfig(ModelConfig): model_name: OpenAIModel = Field(default=default_openai_model, # type: ignore description="The name of the model to use.") - max_tokens: PositiveInt = Field(1000, description="""\ + max_tokens: PositiveInt = Field(4000, description="""\ The maximum number of [tokens](https://platform.openai.com/tokenizer) to generate in the chat completion. The total length of input tokens and generated tokens is limited by the model's context length. \ From 87a66a733b5e58b3301a54d0e1c709fc441ae4c6 Mon Sep 17 00:00:00 2001 From: leon Date: Tue, 3 Dec 2024 18:38:38 +0100 Subject: [PATCH 08/18] Increase max_input_tokens to 5000 and update element_ids assignment in feedback model conversion --- .../modeling/module_modeling_llm/module_modeling_llm/config.py | 2 +- .../utils/convert_to_athana_feedback_model.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/config.py b/modules/modeling/module_modeling_llm/module_modeling_llm/config.py index 169ee047a..c7f0f4ad0 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/config.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/config.py @@ -43,7 +43,7 @@ class GenerateSuggestionsPrompt(BaseModel): class BasicApproachConfig(BaseModel): """This approach uses a LLM with a single prompt to generate feedback in a single step.""" - max_input_tokens: int = Field(default=3000, description="Maximum number of tokens in the input prompt.") + max_input_tokens: int = Field(default=5000, description="Maximum number of tokens in the input prompt.") model: ModelConfigType = Field(default=DefaultModelConfig()) # type: ignore generate_suggestions_prompt: GenerateSuggestionsPrompt = Field(default=GenerateSuggestionsPrompt()) diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py index 0a6b260c5..4ade6b3c7 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/utils/convert_to_athana_feedback_model.py @@ -44,7 +44,7 @@ def convert_to_athana_feedback_model( id=None, is_graded=False, reference=reference, - element_ids=[reference_id] if reference_id else [] + element_ids=[reference] if reference else [] # Todo: Remove after adding migrations to athena )) return feedbacks \ No newline at end of file From 322fd5567ab61a67b03f06f0b1ffd11b3b5ec9af Mon Sep 17 00:00:00 2001 From: leon Date: Fri, 6 Dec 2024 10:40:58 +0100 Subject: [PATCH 09/18] Refactor exercise models to implement polymorphism and establish relationships; fix foreign key references and ensure proper inheritance structure. --- athena/athena/app.py | 2 +- athena/athena/models/db_exercise.py | 17 ++++++++++++++--- athena/athena/models/db_modeling_exercise.py | 19 +++++++++++-------- .../athena/models/db_programming_exercise.py | 12 +++++++++--- .../models/db_structured_grading_criterion.py | 2 +- athena/athena/models/db_text_exercise.py | 12 +++++++++--- athena/athena/storage/exercise_storage.py | 3 +++ 7 files changed, 48 insertions(+), 19 deletions(-) diff --git a/athena/athena/app.py b/athena/athena/app.py index b73cc045d..e9d653f03 100644 --- a/athena/athena/app.py +++ b/athena/athena/app.py @@ -69,4 +69,4 @@ async def validation_exception_handler(request: Request, exc: RequestValidationE return JSONResponse( status_code=422, content={"detail": exc.errors()}, - ) + ) \ No newline at end of file diff --git a/athena/athena/models/db_exercise.py b/athena/athena/models/db_exercise.py index 13fd5593e..d0c5e4909 100644 --- a/athena/athena/models/db_exercise.py +++ b/athena/athena/models/db_exercise.py @@ -1,18 +1,29 @@ from sqlalchemy import Column, String, Float, JSON, Enum as SqlEnum - +from sqlalchemy.orm import relationship +from athena.database import Base from athena.schemas import ExerciseType from .model import Model from .big_integer_with_autoincrement import BigIntegerWithAutoincrement -class DBExercise(Model): +class DBExercise(Model, Base): + __tablename__ = "exercise" id = Column(BigIntegerWithAutoincrement, primary_key=True, index=True, nullable=False) lms_url = Column(String, index=True, nullable=False) title = Column(String, index=True, nullable=False) - type = Column(SqlEnum(ExerciseType), index=True, nullable=False) max_points = Column(Float, index=True, nullable=False) bonus_points = Column(Float, index=True, nullable=False) grading_instructions = Column(String) problem_statement = Column(String) grading_criteria = Column(JSON, nullable=True) meta = Column(JSON, nullable=False) + + structured_grading_criterion = relationship("DBStructuredGradingCriterion", back_populates="exercise", uselist=False) # Use uselist=False for one-to-one + + # Polymorphism, discriminator attribute + type = Column(SqlEnum(ExerciseType), index=True, nullable=False) + + __mapper_args__ = { + 'polymorphic_identity': 'exercise', + 'polymorphic_on': 'type' + } \ No newline at end of file diff --git a/athena/athena/models/db_modeling_exercise.py b/athena/athena/models/db_modeling_exercise.py index 450b32e77..818724ab8 100644 --- a/athena/athena/models/db_modeling_exercise.py +++ b/athena/athena/models/db_modeling_exercise.py @@ -1,14 +1,17 @@ -from sqlalchemy import Column, String +from athena.schemas.exercise_type import ExerciseType +from sqlalchemy import Column, String, ForeignKey from sqlalchemy.orm import relationship - -from athena.database import Base from .db_exercise import DBExercise - - -class DBModelingExercise(DBExercise, Base): +from .big_integer_with_autoincrement import BigIntegerWithAutoincrement +class DBModelingExercise(DBExercise): __tablename__ = "modeling_exercises" - example_solution: str = Column(String) # type: ignore - + example_solution = Column(String) # type: ignore + + id = Column(BigIntegerWithAutoincrement, ForeignKey('exercise.id'), primary_key=True) submissions = relationship("DBModelingSubmission", back_populates="exercise") feedbacks = relationship("DBModelingFeedback", back_populates="exercise") + + __mapper_args__ = { + 'polymorphic_identity': ExerciseType.modeling.value + } \ No newline at end of file diff --git a/athena/athena/models/db_programming_exercise.py b/athena/athena/models/db_programming_exercise.py index b558dd58f..3060e5831 100644 --- a/athena/athena/models/db_programming_exercise.py +++ b/athena/athena/models/db_programming_exercise.py @@ -1,13 +1,15 @@ -from sqlalchemy import Column, String +from athena.schemas.exercise_type import ExerciseType +from sqlalchemy import Column, String, ForeignKey from sqlalchemy.orm import relationship -from athena.database import Base from .db_exercise import DBExercise +from .big_integer_with_autoincrement import BigIntegerWithAutoincrement -class DBProgrammingExercise(DBExercise, Base): +class DBProgrammingExercise(DBExercise): __tablename__ = "programming_exercises" + id = Column(BigIntegerWithAutoincrement, ForeignKey('exercise.id'), primary_key=True) programming_language: str = Column(String, nullable=False) # type: ignore solution_repository_uri: str = Column(String, nullable=False) # type: ignore template_repository_uri: str = Column(String, nullable=False) # type: ignore @@ -15,3 +17,7 @@ class DBProgrammingExercise(DBExercise, Base): submissions = relationship("DBProgrammingSubmission", back_populates="exercise") feedbacks = relationship("DBProgrammingFeedback", back_populates="exercise") + + __mapper_args__ = { + 'polymorphic_identity': ExerciseType.programming.value + } \ No newline at end of file diff --git a/athena/athena/models/db_structured_grading_criterion.py b/athena/athena/models/db_structured_grading_criterion.py index 897e54c69..b0679ae70 100644 --- a/athena/athena/models/db_structured_grading_criterion.py +++ b/athena/athena/models/db_structured_grading_criterion.py @@ -9,7 +9,7 @@ class DBStructuredGradingCriterion(Base): __tablename__ = "structured_grading_criterion" id = Column(BigIntegerWithAutoincrement, primary_key=True, index=True, autoincrement=True) - exercise_id = Column(BigIntegerWithAutoincrement, ForeignKey("exercises.id", ondelete="CASCADE"), index=True, unique=True) # Only one cached instruction per exercise + exercise_id = Column(BigIntegerWithAutoincrement, ForeignKey("exercise.id", ondelete="CASCADE"), index=True, unique=True) # Only one cached instruction per exercise instructions_hash = Column(String, nullable=False) structured_grading_criterion = Column(JSON, nullable=False) lms_url = Column(String, nullable=False) diff --git a/athena/athena/models/db_text_exercise.py b/athena/athena/models/db_text_exercise.py index f3c2823ef..05df9e5c0 100644 --- a/athena/athena/models/db_text_exercise.py +++ b/athena/athena/models/db_text_exercise.py @@ -1,14 +1,20 @@ -from sqlalchemy import Column, String +from athena.schemas.exercise_type import ExerciseType +from sqlalchemy import Column, String, ForeignKey from sqlalchemy.orm import relationship -from athena.database import Base from .db_exercise import DBExercise +from .big_integer_with_autoincrement import BigIntegerWithAutoincrement -class DBTextExercise(DBExercise, Base): +class DBTextExercise(DBExercise): __tablename__ = "text_exercises" + id = Column(BigIntegerWithAutoincrement, ForeignKey('exercise.id'), primary_key=True) example_solution: str = Column(String) # type: ignore submissions = relationship("DBTextSubmission", back_populates="exercise") feedbacks = relationship("DBTextFeedback", back_populates="exercise") + + __mapper_args__ = { + 'polymorphic_identity': ExerciseType.text.value + } \ No newline at end of file diff --git a/athena/athena/storage/exercise_storage.py b/athena/athena/storage/exercise_storage.py index 77ba205ca..f452fc00c 100644 --- a/athena/athena/storage/exercise_storage.py +++ b/athena/athena/storage/exercise_storage.py @@ -42,10 +42,13 @@ def store_exercises(exercises: List[Exercise], lms_url: Optional[str] = None): if lms_url is None: lms_url = get_lms_url() + print("Exercise lms_url 2: ", lms_url, "\n\n\n\n") + with get_db() as db: for e in exercises: exercise_model = e.to_model() exercise_model.lms_url = lms_url + print("Exercise model: ", exercise_model.lms_url, "\n\n\n\n") db.merge(exercise_model) db.commit() From a83ac8b5f9d5e181d451cbb23662da7c2601cfae Mon Sep 17 00:00:00 2001 From: leon Date: Fri, 6 Dec 2024 11:10:07 +0100 Subject: [PATCH 10/18] Refactor exercise storage and structured grading criterion handling; remove debug prints, update caching logic, and change serialization method for structured grading instructions --- athena/athena/storage/exercise_storage.py | 3 --- .../structured_grading_criterion_storage.py | 19 +++++++++---------- .../get_structured_grading_instructions.py | 1 + 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/athena/athena/storage/exercise_storage.py b/athena/athena/storage/exercise_storage.py index f452fc00c..77ba205ca 100644 --- a/athena/athena/storage/exercise_storage.py +++ b/athena/athena/storage/exercise_storage.py @@ -42,13 +42,10 @@ def store_exercises(exercises: List[Exercise], lms_url: Optional[str] = None): if lms_url is None: lms_url = get_lms_url() - print("Exercise lms_url 2: ", lms_url, "\n\n\n\n") - with get_db() as db: for e in exercises: exercise_model = e.to_model() exercise_model.lms_url = lms_url - print("Exercise model: ", exercise_model.lms_url, "\n\n\n\n") db.merge(exercise_model) db.commit() diff --git a/athena/athena/storage/structured_grading_criterion_storage.py b/athena/athena/storage/structured_grading_criterion_storage.py index 0abd1b3f2..459e4c02d 100644 --- a/athena/athena/storage/structured_grading_criterion_storage.py +++ b/athena/athena/storage/structured_grading_criterion_storage.py @@ -6,15 +6,14 @@ from athena.schemas import StructuredGradingCriterion def get_structured_grading_criterion(exercise_id: int, current_hash: Optional[str] = None) -> Optional[StructuredGradingCriterion]: - lms_url = get_lms_url() - with get_db() as db: - cache_entry = db.query(DBStructuredGradingCriterion).filter_by( - exercise_id=exercise_id, lms_url=lms_url - ).first() - if cache_entry is not None and (current_hash is None or cache_entry.instructions_hash.is_(current_hash)): - return StructuredGradingCriterion.parse_raw(cache_entry.cached_instructions) - - return None + lms_url = get_lms_url() + with get_db() as db: + cache_entry = db.query(DBStructuredGradingCriterion).filter_by( + exercise_id=exercise_id, lms_url=lms_url + ).first() + if cache_entry is not None and (current_hash is None or cache_entry.instructions_hash == current_hash): # type: ignore + return StructuredGradingCriterion.parse_obj(cache_entry.structured_grading_criterion) + return None def store_structured_grading_criterion( exercise_id: int, hash: str, structured_instructions: StructuredGradingCriterion @@ -26,7 +25,7 @@ def store_structured_grading_criterion( exercise_id=exercise_id, lms_url=lms_url, instructions_hash=hash, - cached_instructions=structured_instructions.json(), + structured_grading_criterion=structured_instructions.dict(), ) ) db.commit() \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py b/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py index 4796ae01a..75b7ded47 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py @@ -28,6 +28,7 @@ async def get_structured_grading_instructions( current_hash = get_grading_instructions_hash(exercise_model) cached_instructions = get_structured_grading_criterion(exercise_model.exercise_id, current_hash) if cached_instructions: + print("Using cached instructions") return cached_instructions chat_prompt = ChatPromptTemplate.from_messages([ From 429248941eba6de1d9ffa171c38a9c9ef70425fb Mon Sep 17 00:00:00 2001 From: leon Date: Sat, 7 Dec 2024 12:46:12 +0100 Subject: [PATCH 11/18] Fix pylint errors --- .../structured_grading_criterion_storage.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/athena/athena/storage/structured_grading_criterion_storage.py b/athena/athena/storage/structured_grading_criterion_storage.py index 459e4c02d..06c3ebdd3 100644 --- a/athena/athena/storage/structured_grading_criterion_storage.py +++ b/athena/athena/storage/structured_grading_criterion_storage.py @@ -6,17 +6,17 @@ from athena.schemas import StructuredGradingCriterion def get_structured_grading_criterion(exercise_id: int, current_hash: Optional[str] = None) -> Optional[StructuredGradingCriterion]: - lms_url = get_lms_url() - with get_db() as db: - cache_entry = db.query(DBStructuredGradingCriterion).filter_by( - exercise_id=exercise_id, lms_url=lms_url - ).first() - if cache_entry is not None and (current_hash is None or cache_entry.instructions_hash == current_hash): # type: ignore - return StructuredGradingCriterion.parse_obj(cache_entry.structured_grading_criterion) - return None + lms_url = get_lms_url() + with get_db() as db: + cache_entry = db.query(DBStructuredGradingCriterion).filter_by( + exercise_id=exercise_id, lms_url=lms_url + ).first() + if cache_entry is not None and (current_hash is None or cache_entry.instructions_hash == current_hash): # type: ignore + return StructuredGradingCriterion.parse_obj(cache_entry.structured_grading_criterion) + return None def store_structured_grading_criterion( - exercise_id: int, hash: str, structured_instructions: StructuredGradingCriterion + exercise_id: int, hash: str, structured_instructions: StructuredGradingCriterion ): lms_url = get_lms_url() with get_db() as db: From f37b8376ae72ea900982ea6a4c10e8b116163765 Mon Sep 17 00:00:00 2001 From: LeonWehrhahn Date: Thu, 2 Jan 2025 16:31:03 +0100 Subject: [PATCH 12/18] Add LLM configuration files; refactor model handeling and prompts --- llm_core/llm_capabilities.yml | 18 +++ llm_core/llm_core/core/predict_and_parse.py | 58 +++++++ .../loaders/llm_capabilities_loader.py | 28 ++++ .../llm_core/loaders/llm_config_loader.py | 64 ++++++++ llm_core/llm_core/loaders/openai_loader.py | 73 +++++++++ llm_core/llm_core/models/__init__.py | 41 ----- llm_core/llm_core/models/llm_config.py | 21 +++ llm_core/llm_core/models/model_config.py | 27 +++- llm_core/llm_core/models/openai.py | 145 ------------------ .../llm_core/models/openai_model_config.py | 137 +++++++++++++++++ .../utils/append_format_instructions.py | 42 +++++ llm_core/llm_core/utils/llm_utils.py | 82 ++++------ llm_core/llm_core/utils/model_factory.py | 33 ++++ llm_core/llm_core/utils/predict_and_parse.py | 66 -------- llm_core/poetry.lock | 13 +- llm_core/pyproject.toml | 2 +- .../module_modeling_llm/llm_config.yml | 5 + .../module_modeling_llm/config.py | 10 +- .../core/filter_feedback.py | 5 +- .../core/generate_suggestions.py | 5 +- .../get_structured_grading_instructions.py | 8 +- .../prompts/filter_feedback_prompt.py | 4 - .../prompts/graded_feedback_prompt.py | 4 - .../structured_grading_instructions_prompt.py | 4 - .../module_programming_llm/llm_config.yml | 5 + .../module_programming_llm/config.py | 7 +- .../generate_graded_suggestions_by_file.py | 11 +- ...generate_non_graded_suggestions_by_file.py | 11 +- .../generate_summary_by_file.py | 12 +- .../split_grading_instructions_by_file.py | 14 +- .../split_problem_statement_by_file.py | 10 +- modules/text/module_text_llm/llm_config.yml | 5 + .../module_text_llm/approach_config.py | 7 +- .../basic_approach/generate_suggestions.py | 12 +- .../chain_of_thought_approach/__init__.py | 9 +- .../generate_suggestions.py | 31 ++-- .../module_text_llm/generate_evaluation.py | 16 +- 37 files changed, 628 insertions(+), 417 deletions(-) create mode 100644 llm_core/llm_capabilities.yml create mode 100644 llm_core/llm_core/core/predict_and_parse.py create mode 100644 llm_core/llm_core/loaders/llm_capabilities_loader.py create mode 100644 llm_core/llm_core/loaders/llm_config_loader.py create mode 100644 llm_core/llm_core/loaders/openai_loader.py create mode 100644 llm_core/llm_core/models/llm_config.py delete mode 100644 llm_core/llm_core/models/openai.py create mode 100644 llm_core/llm_core/models/openai_model_config.py create mode 100644 llm_core/llm_core/utils/append_format_instructions.py create mode 100644 llm_core/llm_core/utils/model_factory.py delete mode 100644 llm_core/llm_core/utils/predict_and_parse.py create mode 100644 modules/modeling/module_modeling_llm/llm_config.yml create mode 100644 modules/programming/module_programming_llm/llm_config.yml create mode 100644 modules/text/module_text_llm/llm_config.yml diff --git a/llm_core/llm_capabilities.yml b/llm_core/llm_capabilities.yml new file mode 100644 index 000000000..33e9569b5 --- /dev/null +++ b/llm_core/llm_capabilities.yml @@ -0,0 +1,18 @@ +defaults: + temperature: 0.7 + top_p: 1.0 + + supports_function_calling: true + supports_structured_output: true + +models: + openai_o1-mini: + max_completion_tokens: 8000 + temperature: 1.0 + top_p: 1.0 + n: 1 + presence_penalty: 0.0 + frequency_penalty: 0.0 + supports_system_messages: false + supports_function_calling: false + supports_structured_output: false \ No newline at end of file diff --git a/llm_core/llm_core/core/predict_and_parse.py b/llm_core/llm_core/core/predict_and_parse.py new file mode 100644 index 000000000..cd1e82bc6 --- /dev/null +++ b/llm_core/llm_core/core/predict_and_parse.py @@ -0,0 +1,58 @@ +from typing import Optional, Type, TypeVar, List +from langchain_core.language_models import BaseLanguageModel +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.pydantic_v1 import BaseModel, ValidationError +from langchain_core.runnables import RunnableSequence +from langchain_core.output_parsers import PydanticOutputParser +from athena import get_experiment_environment +from llm_core.utils.append_format_instructions import append_format_instructions +from llm_core.utils.llm_utils import remove_system_message +from llm_core.models.model_config import ModelConfig + +T = TypeVar("T", bound=BaseModel) + +async def predict_and_parse( + model: ModelConfig, + chat_prompt: ChatPromptTemplate, + prompt_input: dict, + pydantic_object: Type[T], + tags: Optional[List[str]], + ) -> Optional[T]: + """ + Predicts an LLM completion using the model and parses the output using the provided Pydantic model + """ + + # Remove system messages if the model does not support them + if not model.supports_system_messages(): + chat_prompt = remove_system_message(chat_prompt) + + llm_model = model.get_model() + + # Add tags + experiment = get_experiment_environment() + tags = tags or [] + if experiment.experiment_id is not None: + tags.append(f"experiment-{experiment.experiment_id}") + if experiment.module_configuration_id is not None: + tags.append(f"module-configuration-{experiment.module_configuration_id}") + if experiment.run_id is not None: + tags.append(f"run-{experiment.run_id}") + + # Currently structured output and function calling both expect the expected json to be in the prompt input + chat_prompt = append_format_instructions(chat_prompt, pydantic_object) + + # Run the model and parse the output + if model.supports_structured_output(): + structured_output_llm = llm_model.with_structured_output(pydantic_object, method = "json_mode") + elif model.supports_function_calling(): + structured_output_llm = llm_model.with_structured_output(pydantic_object) + else: + structured_output_llm = RunnableSequence(llm_model, PydanticOutputParser(pydantic_object=pydantic_object)) + + chain = RunnableSequence(chat_prompt, structured_output_llm) + + try: + return await chain.ainvoke(prompt_input, config={"tags": tags}, debug=True) + except ValidationError as e: + raise ValueError(f"Could not parse output: {e}") from e + diff --git a/llm_core/llm_core/loaders/llm_capabilities_loader.py b/llm_core/llm_core/loaders/llm_capabilities_loader.py new file mode 100644 index 000000000..138fb7350 --- /dev/null +++ b/llm_core/llm_core/loaders/llm_capabilities_loader.py @@ -0,0 +1,28 @@ +import os +import yaml +from pathlib import Path +from typing import Dict + + +CONFIG_PATH = Path(__file__).resolve().parents[2] / "llm_capabilities.yml" +if os.path.exists(CONFIG_PATH): + with open(CONFIG_PATH, "r", encoding="utf-8") as f: + _yaml_config = yaml.safe_load(f) +else: + _yaml_config = {} + +DEFAULTS = _yaml_config.get("defaults", {}) +MODEL_OVERRIDES = _yaml_config.get("models", {}) + + +def get_model_capabilities(model_key: str) -> Dict: + """ + Return the merged dictionary of defaults and overrides + for the given model key. + """ + caps = dict(DEFAULTS) # start with a copy of the defaults + if model_key in MODEL_OVERRIDES: + # override with model-specific entries + for k, v in MODEL_OVERRIDES[model_key].items(): + caps[k] = v + return caps \ No newline at end of file diff --git a/llm_core/llm_core/loaders/llm_config_loader.py b/llm_core/llm_core/loaders/llm_config_loader.py new file mode 100644 index 000000000..f4044d946 --- /dev/null +++ b/llm_core/llm_core/loaders/llm_config_loader.py @@ -0,0 +1,64 @@ +from llm_core.models.llm_config import LLMConfig, LLMConfigModel, RawLLMConfig +from llm_core.utils.model_factory import create_config_for_model +import yaml +from pathlib import Path +from typing import Dict, Optional + +_state: Dict[str, Optional[LLMConfig]] = {"llm_config": None} + +def _load_raw_llm_config(path: Optional[str] = None) -> RawLLMConfig: + """ + Loads the LLM configuration from a YAML file and returns a validated LLMConfig. + Raises pydantic.ValidationError if something is incorrect in the YAML. + """ + # By default, we assume 'llm_config.yml' is in the same directory + if path is None: + path = "llm_config.yml" + + config_path = Path(path).resolve() + if not config_path.exists(): + raise FileNotFoundError(f"LLM config file not found at: {config_path}") + + with config_path.open("r", encoding="utf-8") as f: + raw_data = yaml.safe_load(f) + + # Validate and parse the Pydantic model + return RawLLMConfig(**raw_data) + +def _materialize_llm_config(raw_config: RawLLMConfig) -> LLMConfig: + base_model = raw_config.models.base_model + if not base_model: + raise ValueError("Missing required 'base_model' in models") + + models_obj = LLMConfigModel( + base_model_config=create_config_for_model(base_model), + mini_model_config=( + create_config_for_model(raw_config.models.mini_model) + if raw_config.models.mini_model + else None + ), + fast_reasoning_model_config=( + create_config_for_model(raw_config.models.fast_reasoning_model) + if raw_config.models.fast_reasoning_model + else None + ), + long_reasoning_model_config=( + create_config_for_model(raw_config.models.long_reasoning_model) + if raw_config.models.long_reasoning_model + else None + ), + ) + + return LLMConfig(models=models_obj) + +def get_llm_config(path: Optional[str] = None) -> LLMConfig: + """ + Public function: returns a cached LLMConfig instance. + If not loaded yet, loads from YAML and materializes it. + """ + # Here we read/write _state["llm_config"] without using 'global'. + if _state["llm_config"] is None: + raw_config = _load_raw_llm_config(path) + _state["llm_config"] = _materialize_llm_config(raw_config) + + return _state["llm_config"] \ No newline at end of file diff --git a/llm_core/llm_core/loaders/openai_loader.py b/llm_core/llm_core/loaders/openai_loader.py new file mode 100644 index 000000000..f04a93ff7 --- /dev/null +++ b/llm_core/llm_core/loaders/openai_loader.py @@ -0,0 +1,73 @@ +import os +from llm_core.loaders.llm_capabilities_loader import get_model_capabilities +import openai +import requests + +from typing import Dict, List +from enum import Enum +from pydantic import Field, PrivateAttr, validator, PositiveInt +from langchain.base_language import BaseLanguageModel +from langchain_openai import AzureChatOpenAI, ChatOpenAI + +from athena.logger import logger + + +# Discover available models from OpenAI / Azure +OPENAI_PREFIX = "openai_" +AZURE_OPENAI_PREFIX = "azure_openai_" +openai_available = bool(os.environ.get("OPENAI_API_KEY")) +azure_openai_available = bool(os.environ.get("AZURE_OPENAI_API_KEY")) + +available_models: Dict[str, BaseLanguageModel] = {} + +# OpenAI +if openai_available: + openai.api_type = "openai" + for model in openai.models.list(): + if "gpt" in model.id or "o1" in model.id: + available_models[OPENAI_PREFIX + model.id] = ChatOpenAI(model=model.id) + + +# Azure OpenAI +if azure_openai_available: + def _get_azure_openai_deployments() -> List[str]: + base_url = f"{os.environ.get('AZURE_OPENAI_ENDPOINT')}/openai" + headers = {"api-key": os.environ["AZURE_OPENAI_API_KEY"]} + + models_response = requests.get( + f"{base_url}/models?api-version=2023-03-15-preview", + headers=headers, timeout=30 + ) + models_data = models_response.json()["data"] + + deployments_response = requests.get( + f"{base_url}/deployments?api-version=2023-03-15-preview", + headers=headers, timeout=30 + ) + deployments_data = deployments_response.json()["data"] + + # If deployment["model"] is substring of model["id"], we consider it a chat completion model + chat_completion_models = ",".join( + m["id"] for m in models_data if m["capabilities"]["chat_completion"] + ) + return [ + d["id"] + for d in deployments_data + if d["model"] in chat_completion_models + ] + + for deployment in _get_azure_openai_deployments(): + available_models[AZURE_OPENAI_PREFIX + deployment] = AzureChatOpenAI(azure_deployment=deployment) + + +if available_models: + logger.info("Available openai models: %s", ", ".join(available_models.keys())) +else: + logger.warning("No openai/azure models discovered.") + +# Enum for referencing the discovered models +if available_models: + OpenAIModel = Enum('OpenAIModel', {name: name for name in available_models}) # type: ignore +else: + class OpenAIModel(str, Enum): + pass \ No newline at end of file diff --git a/llm_core/llm_core/models/__init__.py b/llm_core/llm_core/models/__init__.py index f5e3edd60..e69de29bb 100644 --- a/llm_core/llm_core/models/__init__.py +++ b/llm_core/llm_core/models/__init__.py @@ -1,41 +0,0 @@ -import os -from typing import Type, Union, List, Optional -from langchain.base_language import BaseLanguageModel - -from llm_core.models.model_config import ModelConfig - - -DefaultModelConfig: Type[ModelConfig] -MiniModelConfig: ModelConfig -default_model_name = os.environ.get("LLM_DEFAULT_MODEL") -evaluation_model_name = os.environ.get("LLM_EVALUATION_MODEL") - -# Model used during evaluation for judging the output (should be a more powerful model) -evaluation_model: Optional[BaseLanguageModel] = None - -types: List[Type[ModelConfig]] = [] -try: - import llm_core.models.openai as openai_config - types.append(openai_config.OpenAIModelConfig) - if default_model_name in openai_config.available_models: - DefaultModelConfig = openai_config.OpenAIModelConfig - if "openai_gpt-4o-mini" in openai_config.available_models: - MiniModelConfig = openai_config.OpenAIModelConfig(model_name="openai_gpt-4o-mini",max_tokens=3000, temperature=0,top_p=0.9,presence_penalty=0,frequency_penalty=0) - if evaluation_model_name in openai_config.available_models: - evaluation_model = openai_config.available_models[evaluation_model_name] -except AttributeError: - pass - -if not types: - raise EnvironmentError( - "No model configurations available, please set up at least one provider in the environment variables.") - -if 'DefaultModelConfig' not in globals(): - DefaultModelConfig = types[0] - -type0 = types[0] -if len(types) == 1: - ModelConfigType = type0 -else: - type1 = types[1] - ModelConfigType = Union[type0, type1] # type: ignore \ No newline at end of file diff --git a/llm_core/llm_core/models/llm_config.py b/llm_core/llm_core/models/llm_config.py new file mode 100644 index 000000000..192588dfc --- /dev/null +++ b/llm_core/llm_core/models/llm_config.py @@ -0,0 +1,21 @@ +from pydantic import BaseModel +from typing import Optional +from llm_core.models.model_config import ModelConfig + +class RawModelsSection(BaseModel): + base_model: Optional[str] + mini_model: Optional[str] + fast_reasoning_model: Optional[str] + long_reasoning_model: Optional[str] + +class RawLLMConfig(BaseModel): + models: RawModelsSection + +class LLMConfigModel(BaseModel): + base_model_config: ModelConfig + mini_model_config: Optional[ModelConfig] + fast_reasoning_model_config: Optional[ModelConfig] + long_reasoning_model_config: Optional[ModelConfig] + +class LLMConfig(BaseModel): + models: LLMConfigModel \ No newline at end of file diff --git a/llm_core/llm_core/models/model_config.py b/llm_core/llm_core/models/model_config.py index f433ab587..574818a19 100644 --- a/llm_core/llm_core/models/model_config.py +++ b/llm_core/llm_core/models/model_config.py @@ -4,7 +4,32 @@ class ModelConfig(BaseModel, ABC): - + """ + Abstract base class representing a general model configuration. + """ + @abstractmethod def get_model(self) -> BaseLanguageModel: pass + + @abstractmethod + def supports_system_messages(self) -> bool: + """ + Indicates whether this model supports system prompt messages + (for example, the 'system' role, in gpt4o) + """ + pass + + @abstractmethod + def supports_function_calling(self) -> bool: + """ + Indicates whether this model supports function-calling + """ + pass + + @abstractmethod + def supports_structured_output(self) -> bool: + """ + Indicates whether this model supports producing structured output + """ + pass \ No newline at end of file diff --git a/llm_core/llm_core/models/openai.py b/llm_core/llm_core/models/openai.py deleted file mode 100644 index 7bcc0f11f..000000000 --- a/llm_core/llm_core/models/openai.py +++ /dev/null @@ -1,145 +0,0 @@ -import os -import openai -import requests - -from typing import Dict, List -from enum import Enum -from pydantic import Field, validator, PositiveInt -from langchain.base_language import BaseLanguageModel -from langchain_openai import AzureChatOpenAI, ChatOpenAI - -from athena.logger import logger -from .model_config import ModelConfig -from .callbacks import UsageHandler - - -OPENAI_PREFIX = "openai_" -AZURE_OPENAI_PREFIX = "azure_openai_" -openai_available = bool(os.environ.get("OPENAI_API_KEY")) -azure_openai_available = bool(os.environ.get("AZURE_OPENAI_API_KEY")) - -available_models: Dict[str, BaseLanguageModel] = {} - -# Load Non-Azure OpenAI models -if openai_available: - openai.api_type = "openai" - for model in openai.models.list(): - if "gpt" in model.id: - available_models[OPENAI_PREFIX + model.id] = ChatOpenAI(model=model.id) - -# Load Azure OpenAI models -if azure_openai_available: - def _get_azure_openai_deployments() -> List[str]: - # If this breaks in the future we have to use azure-mgmt-cognitiveservices which needs 6 additional environment variables - base_url = f"{os.environ.get('AZURE_OPENAI_ENDPOINT')}/openai" - headers = { - "api-key": os.environ["AZURE_OPENAI_API_KEY"] - } - - models_response = requests.get(f"{base_url}/models?api-version=2023-03-15-preview", headers=headers, timeout=30) - models_data = models_response.json()["data"] - deployments_response = requests.get(f"{base_url}/deployments?api-version=2023-03-15-preview", headers=headers, - timeout=30) - deployments_data = deployments_response.json()["data"] - - # Check if deployment["model"] is a substring of model["id"], i.e. "gpt-4o" is substring "gpt-4o-2024-05-13" - chat_completion_models = ",".join(model["id"] for model in models_data if model["capabilities"]["chat_completion"]) - return [deployment["id"] for deployment in deployments_data if deployment["model"] in chat_completion_models] - - for deployment in _get_azure_openai_deployments(): - available_models[AZURE_OPENAI_PREFIX + deployment] = AzureChatOpenAI(azure_deployment=deployment) - -if available_models: - logger.info("Available openai models: %s", ", ".join(available_models.keys())) - - OpenAIModel = Enum('OpenAIModel', {name: name for name in available_models}) # type: ignore - default_model_name = None - if "LLM_DEFAULT_MODEL" in os.environ and os.environ["LLM_DEFAULT_MODEL"] in available_models: - default_model_name = os.environ["LLM_DEFAULT_MODEL"] - if default_model_name not in available_models: - default_model_name = list(available_models.keys())[0] - - default_openai_model = OpenAIModel[default_model_name] # type: ignore - - # Long descriptions will be displayed in the playground UI and are copied from the OpenAI docs - class OpenAIModelConfig(ModelConfig): - """OpenAI LLM configuration.""" - - model_name: OpenAIModel = Field(default=default_openai_model, # type: ignore - description="The name of the model to use.") - max_tokens: PositiveInt = Field(4000, description="""\ -The maximum number of [tokens](https://platform.openai.com/tokenizer) to generate in the chat completion. - -The total length of input tokens and generated tokens is limited by the model's context length. \ -[Example Python code](https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb) for counting tokens.\ -""") - - temperature: float = Field(default=0.0, ge=0, le=2, description="""\ -What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, \ -while lower values like 0.2 will make it more focused and deterministic. - -We generally recommend altering this or `top_p` but not both.\ -""") - - top_p: float = Field(default=1, ge=0, le=1, description="""\ -An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. \ -So 0.1 means only the tokens comprising the top 10% probability mass are considered. - -We generally recommend altering this or `temperature` but not both.\ -""") - - presence_penalty: float = Field(default=0, ge=-2, le=2, description="""\ -Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, \ -increasing the model's likelihood to talk about new topics. - -[See more information about frequency and presence penalties.](https://platform.openai.com/docs/api-reference/parameter-details)\ -""") - - frequency_penalty: float = Field(default=0, ge=-2, le=2, description="""\ -Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, \ -decreasing the model's likelihood to repeat the same line verbatim. - -[See more information about frequency and presence penalties.](https://platform.openai.com/docs/api-reference/parameter-details)\ -""") - - @validator('max_tokens') - def max_tokens_must_be_positive(cls, v): - """ - Validate that max_tokens is a positive integer. - """ - if v <= 0: - raise ValueError('max_tokens must be a positive integer') - return v - - def get_model(self) -> BaseLanguageModel: - """Get the model from the configuration. - - Returns: - BaseLanguageModel: The model. - """ - model = available_models[self.model_name.value] - kwargs = model.__dict__ - secrets = {secret: getattr(model, secret) for secret in model.lc_secrets.keys()} - kwargs.update(secrets) - - model_kwargs = kwargs.get("model_kwargs", {}) - for attr, value in self.dict().items(): - if attr == "model_name": - # Skip model_name - continue - if hasattr(model, attr): - # If the model has the attribute, add it to kwargs - kwargs[attr] = value - else: - # Otherwise, add it to model_kwargs (necessary for chat models) - model_kwargs[attr] = value - kwargs["model_kwargs"] = model_kwargs - kwargs["callbacks"] = [UsageHandler()] - - # Initialize a copy of the model using the config - model = model.__class__(**kwargs) - return model - - - class Config: - title = 'OpenAI' \ No newline at end of file diff --git a/llm_core/llm_core/models/openai_model_config.py b/llm_core/llm_core/models/openai_model_config.py new file mode 100644 index 000000000..443942ffb --- /dev/null +++ b/llm_core/llm_core/models/openai_model_config.py @@ -0,0 +1,137 @@ +from llm_core.loaders.openai_loader import OpenAIModel +from llm_core.loaders.llm_capabilities_loader import get_model_capabilities +from pydantic import Field, PrivateAttr, validator, PositiveInt +from langchain.base_language import BaseLanguageModel +from llm_core.loaders.openai_loader import available_models + +from .model_config import ModelConfig +from .callbacks import UsageHandler + +class OpenAIModelConfig(ModelConfig): + """OpenAI LLM configuration.""" + + model_name: OpenAIModel = Field( + description="The name of the model to use (e.g., openai_gpt-3.5-turbo)." + ) + + max_completion_tokens: PositiveInt = Field( + 4000, + description="An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens." + ) + + temperature: float = Field( + default=0.0, ge=0, le=2, + description="""\ +What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, +while lower values like 0.2 will make it more focused and deterministic. + +We generally recommend altering this or `top_p` but not both.\ +""" + ) + + top_p: float = Field( + default=1, ge=0, le=1, + description="""\ +An alternative to sampling with temperature, called nucleus sampling, where the model considers the results +of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability +mass are considered. + +We generally recommend altering this or `temperature` but not both.\ +""" + ) + + presence_penalty: float = Field( + default=0, ge=-2, le=2, + description="""\ +Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, +increasing the model's likelihood to talk about new topics. + +[See more information about frequency and presence penalties.](https://platform.openai.com/docs/api-reference/parameter-details)\ +""" + ) + + frequency_penalty: float = Field( + default=0, ge=-2, le=2, + description="""\ +Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, +decreasing the model's likelihood to repeat the same line verbatim. + +[See more information about frequency and presence penalties.](https://platform.openai.com/docs/api-reference/parameter-details)\ +""" + ) + + # Private boolean flags + _supports_function_calling: bool = PrivateAttr(default=True) + _supports_structured_output: bool = PrivateAttr(default=True) + _supports_system_messages: bool = PrivateAttr(default=True) + + # Initialization: Merge YAML capabilities + def __init__(self, **data): + super().__init__(**data) + raw_key = self.model_name.value if hasattr(self.model_name, "value") else self.model_name + caps = get_model_capabilities(raw_key) + + fields_set_by_user = set(data.keys()) + + # If user didn't pass a certain field, override from YAML + if "temperature" not in fields_set_by_user and "temperature" in caps: + self.temperature = float(caps["temperature"]) + if "top_p" not in fields_set_by_user and "top_p" in caps: + self.top_p = float(caps["top_p"]) + if "presence_penalty" not in fields_set_by_user and "presence_penalty" in caps: + self.presence_penalty = float(caps["presence_penalty"]) + if "frequency_penalty" not in fields_set_by_user and "frequency_penalty" in caps: + self.frequency_penalty = float(caps["frequency_penalty"]) + if "max_completion_tokens" not in fields_set_by_user and "max_completion_tokens" in caps: + self.max_completion_tokens = caps["max_completion_tokens"] + + # set booleans from caps + self._supports_function_calling = bool(caps.get("supports_function_calling", True)) + self._supports_structured_output = bool(caps.get("supports_structured_output", True)) + self._supports_system_messages = bool(caps.get("supports_system_messages", True)) + + def supports_system_messages(self) -> bool: + return self._supports_system_messages + + def supports_function_calling(self) -> bool: + return self._supports_function_calling + + def supports_structured_output(self) -> bool: + return self._supports_structured_output + + @validator('max_completion_tokens') + def max_completion_tokens_must_be_positive(cls, v): + if v <= 0: + raise ValueError('max_completion_tokens must be a positive integer') + return v + + def get_model(self) -> BaseLanguageModel: + """ + Create the ChatOpenAI/AzureChatOpenAI object. + Only add parameters that the model actually supports. + """ + model = available_models[self.model_name.value] + kwargs = model.__dict__.copy() + + secrets = {secret: getattr(model, secret) for secret in model.lc_secrets.keys()} + kwargs.update(secrets) + + model_kwargs = kwargs.get("model_kwargs", {}) + + for attr, value in self.dict().items(): + if attr == "model_name": + continue + if hasattr(model, attr): + kwargs[attr] = value + else: + model_kwargs[attr] = value + + + kwargs["model_kwargs"] = model_kwargs + kwargs["callbacks"] = [UsageHandler()] + + new_model = model.__class__(**kwargs) + return new_model + + class Config: + title = 'OpenAI' \ No newline at end of file diff --git a/llm_core/llm_core/utils/append_format_instructions.py b/llm_core/llm_core/utils/append_format_instructions.py new file mode 100644 index 000000000..8cf44bec8 --- /dev/null +++ b/llm_core/llm_core/utils/append_format_instructions.py @@ -0,0 +1,42 @@ +from typing import Type, TypeVar +from langchain.prompts import ( + ChatPromptTemplate, + SystemMessagePromptTemplate, + HumanMessagePromptTemplate, +) +from langchain.output_parsers import PydanticOutputParser +from pydantic import BaseModel + +T = TypeVar("T", bound=BaseModel) + +def append_format_instructions( + chat_prompt: ChatPromptTemplate, + pydantic_object: Type[T] +) -> ChatPromptTemplate: + """ + Appends format instructions to the first message of an existing chat prompt + """ + output_parser = PydanticOutputParser(pydantic_object=pydantic_object) + format_instructions = output_parser.get_format_instructions() + + messages = chat_prompt.messages + if not messages: + raise ValueError("Cannot append format instructions to an empty chat prompt") + + # Append format instructions to the first message + first_message = messages[0] + if isinstance(first_message, SystemMessagePromptTemplate): + new_first_message = SystemMessagePromptTemplate.from_template( + first_message.prompt.template + "\n\n{format_instructions}" # type: ignore[attr-defined] + ) + elif isinstance(first_message, HumanMessagePromptTemplate): + new_first_message = HumanMessagePromptTemplate.from_template( + first_message.prompt.template + "\n\n{format_instructions}" # type: ignore[attr-defined] + ) + else: + raise ValueError(f"Unsupported message type for appending format instructions: {type(first_message)}") + + new_first_message.prompt.partial_variables = {"format_instructions": format_instructions} # type: ignore[attr-defined] + new_first_message.prompt.input_variables.remove("format_instructions") # type: ignore[attr-defined] + + return ChatPromptTemplate.from_messages([new_first_message] + messages[1:]) diff --git a/llm_core/llm_core/utils/llm_utils.py b/llm_core/llm_core/utils/llm_utils.py index 0bfeb047d..227d18bf0 100644 --- a/llm_core/llm_core/utils/llm_utils.py +++ b/llm_core/llm_core/utils/llm_utils.py @@ -1,19 +1,15 @@ -from typing import Type, TypeVar, List +from typing import List, TypeVar from pydantic import BaseModel import tiktoken -from langchain_openai import AzureChatOpenAI, ChatOpenAI -from langchain.base_language import BaseLanguageModel from langchain.prompts import ( ChatPromptTemplate, SystemMessagePromptTemplate, HumanMessagePromptTemplate, ) -from langchain.output_parsers import PydanticOutputParser from athena import emit_meta T = TypeVar("T", bound=BaseModel) - def num_tokens_from_string(string: str) -> int: """Returns the number of tokens in a text string.""" encoding = tiktoken.get_encoding("cl100k_base") @@ -26,11 +22,13 @@ def num_tokens_from_prompt(chat_prompt: ChatPromptTemplate, prompt_input: dict) return num_tokens_from_string(chat_prompt.format(**prompt_input)) -def check_prompt_length_and_omit_features_if_necessary(prompt: ChatPromptTemplate, - prompt_input: dict, - max_input_tokens: int, - omittable_features: List[str], - debug: bool): +def check_prompt_length_and_omit_features_if_necessary( + prompt: ChatPromptTemplate, + prompt_input: dict, + max_input_tokens: int, + omittable_features: List[str], + debug: bool, +): """Check if the input is too long and omit features if necessary. Note: Omitted features will be replaced with "omitted" in the prompt @@ -43,7 +41,7 @@ def check_prompt_length_and_omit_features_if_necessary(prompt: ChatPromptTemplat debug (bool): Debug flag Returns: - (dict, bool): Tuple of (prompt_input, should_run) where prompt_input is the input with omitted features and + (dict, bool): Tuple of (prompt_input, should_run) where prompt_input is the input with omitted features and should_run is True if the model should run, False otherwise """ if num_tokens_from_prompt(prompt, prompt_input) <= max_input_tokens: @@ -66,45 +64,29 @@ def check_prompt_length_and_omit_features_if_necessary(prompt: ChatPromptTemplat return prompt_input, False -def supports_function_calling(model: BaseLanguageModel): - """Returns True if the model supports function calling, False otherwise - - Args: - model (BaseLanguageModel): The model to check +def get_chat_prompt( + system_message: str, + human_message: str, +) -> ChatPromptTemplate: + system_message_prompt = SystemMessagePromptTemplate.from_template(system_message) + human_message_prompt = HumanMessagePromptTemplate.from_template(human_message) + return ChatPromptTemplate.from_messages( + [system_message_prompt, human_message_prompt] + ) - Returns: - boolean: True if the model supports function calling, False otherwise +def remove_system_message(chat_prompt: ChatPromptTemplate) -> ChatPromptTemplate: """ - return isinstance(model, ChatOpenAI) or isinstance(model, AzureChatOpenAI) - - -def get_chat_prompt_with_formatting_instructions( - model: BaseLanguageModel, - system_message: str, - human_message: str, - pydantic_object: Type[T] - ) -> ChatPromptTemplate: - """Returns a ChatPromptTemplate with formatting instructions (if necessary) - - Note: Does nothing if the model supports function calling - - Args: - model (BaseLanguageModel): The model to check if it supports function calling - system_message (str): System message - human_message (str): Human message - pydantic_object (Type[T]): Model to parse the output - - Returns: - ChatPromptTemplate: ChatPromptTemplate with formatting instructions (if necessary) + Create a NEW ChatPromptTemplate with "system" messages replaced as "human" if the model does not support system messages. """ - if supports_function_calling(model): - system_message_prompt = SystemMessagePromptTemplate.from_template(system_message) - human_message_prompt = HumanMessagePromptTemplate.from_template(human_message) - return ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt]) - - output_parser = PydanticOutputParser(pydantic_object=pydantic_object) - system_message_prompt = SystemMessagePromptTemplate.from_template(system_message + "\n{format_instructions}") - system_message_prompt.prompt.partial_variables = {"format_instructions": output_parser.get_format_instructions()} - system_message_prompt.prompt.input_variables.remove("format_instructions") - human_message_prompt = HumanMessagePromptTemplate.from_template(human_message + "\n\nJSON response following the provided schema:") - return ChatPromptTemplate.from_messages([system_message_prompt, human_message_prompt]) \ No newline at end of file + new_prompt_templates = [] + for prompt_msg in chat_prompt.messages: + if isinstance(prompt_msg, SystemMessagePromptTemplate): + # Convert it to a human prompt, preserving content + new_prompt_templates.append( + HumanMessagePromptTemplate.from_template( + "[System message]: " + prompt_msg.prompt.template # type: ignore + ) + ) + else: + new_prompt_templates.append(prompt_msg) + return ChatPromptTemplate.from_messages(new_prompt_templates) diff --git a/llm_core/llm_core/utils/model_factory.py b/llm_core/llm_core/utils/model_factory.py new file mode 100644 index 000000000..097cbd06f --- /dev/null +++ b/llm_core/llm_core/utils/model_factory.py @@ -0,0 +1,33 @@ + +from typing import Optional +from llm_core.models.openai_model_config import OpenAIModelConfig + +def find_provider_for_model(model_name: str) -> Optional[str]: + """ + Determine which provider a model_name belongs to based on its prefix or a naming pattern. + """ + model_name_lower = model_name.lower() + + if model_name_lower.startswith("openai_"): + return "openai" + elif model_name_lower.startswith("azure_openai_"): + return "azure_openai" + elif model_name_lower.startswith("replicate_"): + return "replicate" + else: + return None + +def create_config_for_model(model_name: str) -> OpenAIModelConfig: # or Union[OpenAIModelConfig, AzureOpenAIModelConfig, ...] + """ + Create the appropriate ModelConfig subclass instance (OpenAI, Azure, etc.) + depending on the provider indicated by `model_name`. + """ + provider = find_provider_for_model(model_name) + if provider == "openai": + return OpenAIModelConfig(model_name=model_name) + elif provider == "azure_openai": + return OpenAIModelConfig(model_name=model_name) + elif provider == "replicate": + raise NotImplementedError("Replicate support is not implemented yet.") + else: + raise ValueError(f"Unknown provider for model name '{model_name}'") \ No newline at end of file diff --git a/llm_core/llm_core/utils/predict_and_parse.py b/llm_core/llm_core/utils/predict_and_parse.py deleted file mode 100644 index e72bb3e91..000000000 --- a/llm_core/llm_core/utils/predict_and_parse.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import Optional, Type, TypeVar, List -from langchain_core.language_models import BaseLanguageModel -from langchain_core.prompts import ChatPromptTemplate -from langchain_core.pydantic_v1 import BaseModel, ValidationError -from langchain_core.runnables import RunnableSequence -from athena import get_experiment_environment - -T = TypeVar("T", bound=BaseModel) - -async def predict_and_parse( - model: BaseLanguageModel, - chat_prompt: ChatPromptTemplate, - prompt_input: dict, - pydantic_object: Type[T], - tags: Optional[List[str]], - use_function_calling: bool = False - ) -> Optional[T]: - """Predicts an LLM completion using the model and parses the output using the provided Pydantic model - - Args: - model (BaseLanguageModel): The model to predict with - chat_prompt (ChatPromptTemplate): Prompt to use - prompt_input (dict): Input parameters to use for the prompt - pydantic_object (Type[T]): Pydantic model to parse the output - tags (Optional[List[str]]: List of tags to tag the prediction with - - Returns: - Optional[T]: Parsed output, or None if it could not be parsed - """ - experiment = get_experiment_environment() - - tags = tags or [] - if experiment.experiment_id is not None: - tags.append(f"experiment-{experiment.experiment_id}") - if experiment.module_configuration_id is not None: - tags.append(f"module-configuration-{experiment.module_configuration_id}") - if experiment.run_id is not None: - tags.append(f"run-{experiment.run_id}") - - - if (use_function_calling): - structured_output_llm = model.with_structured_output(pydantic_object) - chain = chat_prompt | structured_output_llm - - try: - result = await chain.ainvoke(prompt_input, config={"tags": tags}) - - if isinstance(result, pydantic_object): - return result - else: - raise ValueError("Parsed output does not match the expected Pydantic model.") - - except ValidationError as e: - raise ValueError(f"Could not parse output: {e}") from e - - else: - structured_output_llm = model.with_structured_output(pydantic_object, method = "json_mode") - chain = RunnableSequence( - chat_prompt, - structured_output_llm - ) - try: - return await chain.ainvoke(prompt_input, config={"tags": tags}) - except ValidationError as e: - raise ValueError(f"Could not parse output: {e}") from e - diff --git a/llm_core/poetry.lock b/llm_core/poetry.lock index 1d67f0b38..a4df631af 100644 --- a/llm_core/poetry.lock +++ b/llm_core/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -1247,13 +1247,13 @@ files = [ [[package]] name = "openai" -version = "1.51.1" +version = "1.58.1" description = "The official Python library for the openai API" optional = false -python-versions = ">=3.7.1" +python-versions = ">=3.8" files = [ - {file = "openai-1.51.1-py3-none-any.whl", hash = "sha256:035ba637bef7523282b5b8d9f2f5fdc0bb5bc18d52af2bfc7f64e4a7b0a169fb"}, - {file = "openai-1.51.1.tar.gz", hash = "sha256:a4908d68e0a1f4bcb45cbaf273c5fbdc3a4fa6239bb75128b58b94f7d5411563"}, + {file = "openai-1.58.1-py3-none-any.whl", hash = "sha256:e2910b1170a6b7f88ef491ac3a42c387f08bd3db533411f7ee391d166571d63c"}, + {file = "openai-1.58.1.tar.gz", hash = "sha256:f5a035fd01e141fc743f4b0e02c41ca49be8fab0866d3b67f5f29b4f4d3c0973"}, ] [package.dependencies] @@ -1268,6 +1268,7 @@ typing-extensions = ">=4.11,<5" [package.extras] datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +realtime = ["websockets (>=13,<15)"] [[package]] name = "orjson" @@ -2460,4 +2461,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.0" python-versions = "3.11.*" -content-hash = "9f0b58825849fe2c5079cfaa0339cb4f1249abc09864cf17999b57dc1a2f087f" +content-hash = "f5b0912af9142a6bce7409aa8ee02b57242f7da97ed916d6cf9b98c06c8c331b" diff --git a/llm_core/pyproject.toml b/llm_core/pyproject.toml index 9c984b508..d9a5c4591 100644 --- a/llm_core/pyproject.toml +++ b/llm_core/pyproject.toml @@ -14,7 +14,7 @@ langchain = "0.2.15" langchain-community="0.2.10" langchain-openai = "0.1.23" nltk = "3.8.1" -openai = "1.51.1" +openai = "1.58.1" python-dotenv = "1.0.0" tiktoken = "0.7.0" diff --git a/modules/modeling/module_modeling_llm/llm_config.yml b/modules/modeling/module_modeling_llm/llm_config.yml new file mode 100644 index 000000000..25842669a --- /dev/null +++ b/modules/modeling/module_modeling_llm/llm_config.yml @@ -0,0 +1,5 @@ +models: + base_model: openai_gpt-4o + mini_model: openai_gpt-4o-mini + fast_reasoning_model: openai_o1-mini + long_reasoning_model: openai_o1-preview \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/config.py b/modules/modeling/module_modeling_llm/module_modeling_llm/config.py index c7f0f4ad0..b3998c7f4 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/config.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/config.py @@ -1,13 +1,16 @@ +from llm_core.loaders.llm_config_loader import get_llm_config from pydantic import BaseModel, Field from athena import config_schema_provider -from llm_core.models import ModelConfigType, DefaultModelConfig +from llm_core.models.model_config import ModelConfig from module_modeling_llm.prompts import ( graded_feedback_prompt, filter_feedback_prompt, structured_grading_instructions_prompt ) +llm_config = get_llm_config() + class GenerateSuggestionsPrompt(BaseModel): """ Features available: **{problem_statement}**, **{example_solution}**, **{grading_instructions}**, **{max_points}**, @@ -44,7 +47,10 @@ class GenerateSuggestionsPrompt(BaseModel): class BasicApproachConfig(BaseModel): """This approach uses a LLM with a single prompt to generate feedback in a single step.""" max_input_tokens: int = Field(default=5000, description="Maximum number of tokens in the input prompt.") - model: ModelConfigType = Field(default=DefaultModelConfig()) # type: ignore + generate_feedback: ModelConfig = Field(default=llm_config.models.base_model_config) + filter_feedback: ModelConfig = Field(default=llm_config.models.base_model_config) + review_feedback: ModelConfig = Field(default=llm_config.models.fast_reasoning_model_config) + generate_grading_instructions: ModelConfig = Field(default=llm_config.models.base_model_config) generate_suggestions_prompt: GenerateSuggestionsPrompt = Field(default=GenerateSuggestionsPrompt()) @config_schema_provider diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/core/filter_feedback.py b/modules/modeling/module_modeling_llm/module_modeling_llm/core/filter_feedback.py index efe2b1871..54021102b 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/core/filter_feedback.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/core/filter_feedback.py @@ -3,7 +3,7 @@ from athena import emit_meta from module_modeling_llm.config import BasicApproachConfig -from llm_core.utils.predict_and_parse import predict_and_parse +from llm_core.core.predict_and_parse import predict_and_parse from module_modeling_llm.models.assessment_model import AssessmentModel from module_modeling_llm.models.exercise_model import ExerciseModel from module_modeling_llm.prompts.filter_feedback_prompt import FilterFeedbackInputs @@ -24,11 +24,10 @@ async def filter_feedback( prompt_inputs = FilterFeedbackInputs( original_feedback=original_feedback.json(), - feedback_output_format=PydanticOutputParser(pydantic_object=AssessmentModel).get_format_instructions() ) feedback_result = await predict_and_parse( - model=config.model.get_model(), # type: ignore[attr-defined] + model=config.filter_feedback, chat_prompt=chat_prompt, prompt_input=prompt_inputs.dict(), pydantic_object=AssessmentModel, diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py b/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py index 61f051348..000958c3d 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/core/generate_suggestions.py @@ -6,7 +6,7 @@ from module_modeling_llm.config import BasicApproachConfig from module_modeling_llm.models.assessment_model import AssessmentModel from module_modeling_llm.prompts.apollon_format_description import apollon_format_description -from llm_core.utils.predict_and_parse import predict_and_parse +from llm_core.core.predict_and_parse import predict_and_parse from module_modeling_llm.prompts.graded_feedback_prompt import GradedFeedbackInputs from module_modeling_llm.models.exercise_model import ExerciseModel @@ -34,7 +34,6 @@ async def generate_suggestions( submission_uml_type=exercise_model.submission_uml_type, example_solution=exercise_model.transformed_example_solution, uml_diagram_format=apollon_format_description, - feedback_output_format=PydanticOutputParser(pydantic_object=AssessmentModel).get_format_instructions() ) chat_prompt = ChatPromptTemplate.from_messages([ @@ -42,7 +41,7 @@ async def generate_suggestions( ("human", config.generate_suggestions_prompt.graded_feedback_human_message)]) feedback_result = await predict_and_parse( - model=config.model.get_model(), # type: ignore[attr-defined] + model=config.generate_feedback, chat_prompt=chat_prompt, prompt_input=prompt_inputs.dict(), pydantic_object=AssessmentModel, diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py b/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py index 75b7ded47..216086c0b 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/core/get_structured_grading_instructions.py @@ -1,14 +1,13 @@ import hashlib import json -from typing import Any, Dict, List, Optional -from athena import logger +from typing import List, Optional from athena.metadata import emit_meta from athena.storage.structured_grading_criterion_storage import get_structured_grading_criterion, store_structured_grading_criterion from langchain_core.output_parsers import PydanticOutputParser from langchain_core.prompts import ChatPromptTemplate from athena.schemas import GradingCriterion, StructuredGradingCriterion -from llm_core.utils.predict_and_parse import predict_and_parse +from llm_core.core.predict_and_parse import predict_and_parse from module_modeling_llm.config import BasicApproachConfig from module_modeling_llm.models.exercise_model import ExerciseModel from module_modeling_llm.prompts.structured_grading_instructions_prompt import StructuredGradingInstructionsInputs @@ -42,11 +41,10 @@ async def get_structured_grading_instructions( grading_instructions=grading_instructions or "No grading instructions.", submission_uml_type=exercise_model.submission_uml_type, example_solution=exercise_model.transformed_example_solution or "No example solution.", - structured_instructions_output_format=PydanticOutputParser(pydantic_object=StructuredGradingCriterion).get_format_instructions() ) grading_instruction_result = await predict_and_parse( - model=config.model.get_model(), # type: ignore[attr-defined] + model=config.generate_grading_instructions, chat_prompt=chat_prompt, prompt_input=prompt_inputs.dict(), pydantic_object=StructuredGradingCriterion, diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/filter_feedback_prompt.py b/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/filter_feedback_prompt.py index ef048ebe2..f14258528 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/filter_feedback_prompt.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/filter_feedback_prompt.py @@ -3,7 +3,6 @@ class FilterFeedbackInputs(BaseModel): original_feedback: str - feedback_output_format: str filter_feedback_system_message = """ @@ -45,9 +44,6 @@ class FilterFeedbackInputs(BaseModel): "description": "Class 'Car' is correctly defined." Remember, the goal is to guide the student towards improvement without providing a complete solution or grading information in the feedback. Also keep the original structure of the feedback. Just modify the title and description values. - - -{feedback_output_format} """ filter_feedback_human_message = """ diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/graded_feedback_prompt.py b/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/graded_feedback_prompt.py index 71662c84b..60d2d28eb 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/graded_feedback_prompt.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/graded_feedback_prompt.py @@ -11,7 +11,6 @@ class GradedFeedbackInputs(BaseModel): submission_uml_type: str example_solution: Optional[str] = None uml_diagram_format: str - feedback_output_format: str graded_feedback_system_message = """ @@ -56,9 +55,6 @@ class GradedFeedbackInputs(BaseModel): The submission uses the following UML diagram format: {uml_diagram_format} - Note: Don't mention elements that have no name, by there articial name: e.g. ##A or ##B, instad just say e.g. the task in ... is missing ... - - -{feedback_output_format} """ graded_feedback_human_message = """ diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/structured_grading_instructions_prompt.py b/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/structured_grading_instructions_prompt.py index 9db3e6bef..55ebc103f 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/structured_grading_instructions_prompt.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/prompts/structured_grading_instructions_prompt.py @@ -8,15 +8,11 @@ class StructuredGradingInstructionsInputs(BaseModel): grading_instructions: str submission_uml_type: str example_solution: str - structured_instructions_output_format: str structured_grading_instructions_system_message = """You are an AI tutor for {submission_uml_type} modeling exercise assessment at a prestigious university. Create a structured grading instruction based on the given grading instructions (important), the solution diagram (if present) and the problem statement. The structured grading instruction should be highly detailed to ensure consistent grading across different tutors. - - -{structured_instructions_output_format} """ structured_grading_instructions_human_message = """ diff --git a/modules/programming/module_programming_llm/llm_config.yml b/modules/programming/module_programming_llm/llm_config.yml new file mode 100644 index 000000000..25842669a --- /dev/null +++ b/modules/programming/module_programming_llm/llm_config.yml @@ -0,0 +1,5 @@ +models: + base_model: openai_gpt-4o + mini_model: openai_gpt-4o-mini + fast_reasoning_model: openai_o1-mini + long_reasoning_model: openai_o1-preview \ No newline at end of file diff --git a/modules/programming/module_programming_llm/module_programming_llm/config.py b/modules/programming/module_programming_llm/module_programming_llm/config.py index 6a632208e..f9fc60ef7 100644 --- a/modules/programming/module_programming_llm/module_programming_llm/config.py +++ b/modules/programming/module_programming_llm/module_programming_llm/config.py @@ -1,9 +1,10 @@ from abc import ABC +from llm_core.loaders.llm_config_loader import get_llm_config from pydantic import BaseModel, Field +from llm_core.models.model_config import ModelConfig from athena import config_schema_provider -from llm_core.models import ModelConfigType, DefaultModelConfig from module_programming_llm.prompts.generate_graded_suggestions_by_file import ( system_message as generate_graded_suggestions_by_file_system_message, human_message as generate_graded_suggestions_by_file_human_message, @@ -29,6 +30,8 @@ human_message as summarize_submission_by_file_human_message, ) +llm_config = get_llm_config() + class SplitProblemStatementsBasePrompt(BaseModel): """Base class for splitting problem statements into file-based ones, providing a structured approach for processing statements.""" @@ -107,7 +110,7 @@ class BasicApproachConfig(BaseModel): """Defines a basic configuration for processing submissions, incorporating problem statement splitting, feedback generation, and file summarization.""" max_input_tokens: int = Field(default=3000, description="Maximum number of tokens in the input prompt.") - model: ModelConfigType = Field(default=DefaultModelConfig()) # type: ignore + model: ModelConfig = Field(default=llm_config.models.base_model_config) max_number_of_files: int = Field(default=25, description="Maximum number of files. If exceeded, it will prioritize the most important ones.") split_problem_statement_by_file_prompt: SplitProblemStatementsBasePrompt = Field(description="To be defined in " "subclasses.") generate_suggestions_by_file_prompt: SplitProblemStatementsBasePrompt = Field(description="To be defined in " "subclasses.") diff --git a/modules/programming/module_programming_llm/module_programming_llm/generate_graded_suggestions_by_file.py b/modules/programming/module_programming_llm/module_programming_llm/generate_graded_suggestions_by_file.py index 4a29b8f6d..0a6f905b2 100644 --- a/modules/programming/module_programming_llm/module_programming_llm/generate_graded_suggestions_by_file.py +++ b/modules/programming/module_programming_llm/module_programming_llm/generate_graded_suggestions_by_file.py @@ -15,10 +15,10 @@ ) from llm_core.utils.llm_utils import ( check_prompt_length_and_omit_features_if_necessary, - get_chat_prompt_with_formatting_instructions, + get_chat_prompt, num_tokens_from_string, ) -from llm_core.utils.predict_and_parse import predict_and_parse +from llm_core.core.predict_and_parse import predict_and_parse from module_programming_llm.helpers.utils import ( get_diff, @@ -64,13 +64,10 @@ async def generate_suggestions_by_file( config: GradedBasicApproachConfig, debug: bool, ) -> List[Feedback]: - model = config.model.get_model() # type: ignore[attr-defined] - chat_prompt = get_chat_prompt_with_formatting_instructions( - model=model, + chat_prompt = get_chat_prompt( system_message=config.generate_suggestions_by_file_prompt.system_message, human_message=config.generate_suggestions_by_file_prompt.human_message, - pydantic_object=AssessmentModel, ) # Get split problem statement and grading instructions by file (if necessary) @@ -264,7 +261,7 @@ async def generate_suggestions_by_file( results: List[Optional[AssessmentModel]] = await asyncio.gather( *[ predict_and_parse( - model=model, + model=config.model, chat_prompt=chat_prompt, prompt_input=prompt_input, pydantic_object=AssessmentModel, diff --git a/modules/programming/module_programming_llm/module_programming_llm/generate_non_graded_suggestions_by_file.py b/modules/programming/module_programming_llm/module_programming_llm/generate_non_graded_suggestions_by_file.py index de654a427..5d1c216ae 100644 --- a/modules/programming/module_programming_llm/module_programming_llm/generate_non_graded_suggestions_by_file.py +++ b/modules/programming/module_programming_llm/module_programming_llm/generate_non_graded_suggestions_by_file.py @@ -13,10 +13,10 @@ ) from llm_core.utils.llm_utils import ( check_prompt_length_and_omit_features_if_necessary, - get_chat_prompt_with_formatting_instructions, + get_chat_prompt, num_tokens_from_string, ) -from llm_core.utils.predict_and_parse import predict_and_parse +from llm_core.core.predict_and_parse import predict_and_parse from module_programming_llm.helpers.utils import ( get_diff, @@ -58,13 +58,10 @@ async def generate_suggestions_by_file( config: NonGradedBasicApproachConfig, debug: bool, ) -> List[Feedback]: - model = config.model.get_model() # type: ignore[attr-defined] - chat_prompt = get_chat_prompt_with_formatting_instructions( - model=model, + chat_prompt = get_chat_prompt( system_message=config.generate_suggestions_by_file_prompt.system_message, human_message=config.generate_suggestions_by_file_prompt.human_message, - pydantic_object=ImprovementModel, ) prompt_inputs: List[dict] = [] @@ -215,7 +212,7 @@ async def generate_suggestions_by_file( results: List[Optional[ImprovementModel]] = await asyncio.gather( *[ predict_and_parse( - model=model, + model=config.model, chat_prompt=chat_prompt, prompt_input=prompt_input, pydantic_object=ImprovementModel, diff --git a/modules/programming/module_programming_llm/module_programming_llm/generate_summary_by_file.py b/modules/programming/module_programming_llm/module_programming_llm/generate_summary_by_file.py index e5ac6aad7..f2a9c6fb7 100644 --- a/modules/programming/module_programming_llm/module_programming_llm/generate_summary_by_file.py +++ b/modules/programming/module_programming_llm/module_programming_llm/generate_summary_by_file.py @@ -10,10 +10,10 @@ from module_programming_llm.config import GradedBasicApproachConfig, BasicApproachConfig from llm_core.utils.llm_utils import ( - get_chat_prompt_with_formatting_instructions, + get_chat_prompt, num_tokens_from_prompt, ) -from llm_core.utils.predict_and_parse import predict_and_parse +from llm_core.core.predict_and_parse import predict_and_parse from module_programming_llm.helpers.utils import ( get_diff, @@ -70,8 +70,6 @@ async def generate_summary_by_file( if "summary" not in prompt.input_variables: return None - model = config.model.get_model() # type: ignore[attr-defined] - template_repo = exercise.get_template_repository() submission_repo = submission.get_repository() @@ -88,11 +86,9 @@ async def generate_summary_by_file( submission_repo, file_filter=lambda file_path: file_path in changed_files_from_template_to_submission, ) - chat_prompt = get_chat_prompt_with_formatting_instructions( - model=model, + chat_prompt = get_chat_prompt( system_message=config.generate_file_summary_prompt.system_message, human_message=config.generate_file_summary_prompt.human_message, - pydantic_object=SolutionSummary, ) prompt_inputs = [] @@ -118,7 +114,7 @@ async def generate_summary_by_file( results: List[Optional[FileDescription]] = await asyncio.gather( *[ predict_and_parse( - model=model, + model=config.model, chat_prompt=chat_prompt, prompt_input=prompt_input, pydantic_object=FileDescription, diff --git a/modules/programming/module_programming_llm/module_programming_llm/split_grading_instructions_by_file.py b/modules/programming/module_programming_llm/module_programming_llm/split_grading_instructions_by_file.py index 08c08924f..d27cd6860 100644 --- a/modules/programming/module_programming_llm/module_programming_llm/split_grading_instructions_by_file.py +++ b/modules/programming/module_programming_llm/module_programming_llm/split_grading_instructions_by_file.py @@ -9,11 +9,11 @@ from module_programming_llm.config import GradedBasicApproachConfig from llm_core.utils.llm_utils import ( - get_chat_prompt_with_formatting_instructions, + get_chat_prompt, num_tokens_from_string, num_tokens_from_prompt, ) -from llm_core.utils.predict_and_parse import predict_and_parse +from llm_core.core.predict_and_parse import predict_and_parse from module_programming_llm.helpers.utils import format_grading_instructions, get_diff @@ -59,9 +59,7 @@ async def split_grading_instructions_by_file( # Return None if the grading instructions are not in the prompt if "grading_instructions" not in prompt.input_variables: return None - - model = config.model.get_model() # type: ignore[attr-defined] - + template_repo = exercise.get_template_repository() solution_repo = exercise.get_solution_repository() submission_repo = submission.get_repository() @@ -74,11 +72,9 @@ async def split_grading_instructions_by_file( src_repo=template_repo, dst_repo=submission_repo, file_path=None, name_only=True ).split("\n") - chat_prompt = get_chat_prompt_with_formatting_instructions( - model=model, + chat_prompt = get_chat_prompt( system_message=config.split_grading_instructions_by_file_prompt.system_message, human_message=config.split_grading_instructions_by_file_prompt.human_message, - pydantic_object=SplitGradingInstructions, ) prompt_input = { @@ -96,7 +92,7 @@ async def split_grading_instructions_by_file( return None split_grading_instructions = await predict_and_parse( - model=model, + model=config.model, chat_prompt=chat_prompt, prompt_input=prompt_input, pydantic_object=SplitGradingInstructions, diff --git a/modules/programming/module_programming_llm/module_programming_llm/split_problem_statement_by_file.py b/modules/programming/module_programming_llm/module_programming_llm/split_problem_statement_by_file.py index aecf516ac..cb2e886cf 100644 --- a/modules/programming/module_programming_llm/module_programming_llm/split_problem_statement_by_file.py +++ b/modules/programming/module_programming_llm/module_programming_llm/split_problem_statement_by_file.py @@ -9,11 +9,11 @@ from module_programming_llm.config import GradedBasicApproachConfig, BasicApproachConfig from llm_core.utils.llm_utils import ( - get_chat_prompt_with_formatting_instructions, + get_chat_prompt, num_tokens_from_string, num_tokens_from_prompt, ) -from llm_core.utils.predict_and_parse import predict_and_parse +from llm_core.core.predict_and_parse import predict_and_parse from module_programming_llm.helpers.utils import get_diff @@ -69,11 +69,9 @@ async def split_problem_statement_by_file( name_only=True ).split("\n") - chat_prompt = get_chat_prompt_with_formatting_instructions( - model=model, + chat_prompt = get_chat_prompt( system_message=config.split_problem_statement_by_file_prompt.system_message, human_message=config.split_problem_statement_by_file_prompt.human_message, - pydantic_object=SplitProblemStatement ) prompt_input = { @@ -98,7 +96,7 @@ async def split_problem_statement_by_file( return None split_problem_statement = await predict_and_parse( - model=model, + model=config.model, chat_prompt=chat_prompt, prompt_input=prompt_input, pydantic_object=SplitProblemStatement, diff --git a/modules/text/module_text_llm/llm_config.yml b/modules/text/module_text_llm/llm_config.yml new file mode 100644 index 000000000..25842669a --- /dev/null +++ b/modules/text/module_text_llm/llm_config.yml @@ -0,0 +1,5 @@ +models: + base_model: openai_gpt-4o + mini_model: openai_gpt-4o-mini + fast_reasoning_model: openai_o1-mini + long_reasoning_model: openai_o1-preview \ No newline at end of file diff --git a/modules/text/module_text_llm/module_text_llm/approach_config.py b/modules/text/module_text_llm/module_text_llm/approach_config.py index 6626fdd85..97cd1d32e 100644 --- a/modules/text/module_text_llm/module_text_llm/approach_config.py +++ b/modules/text/module_text_llm/module_text_llm/approach_config.py @@ -1,15 +1,18 @@ from abc import ABC +from llm_core.loaders.llm_config_loader import get_llm_config from pydantic import BaseModel, Field -from llm_core.models import ModelConfigType, DefaultModelConfig +from llm_core.models.model_config import ModelConfig from enum import Enum +llm_config = get_llm_config() + class ApproachType(str, Enum): basic = "BasicApproach" chain_of_thought = "ChainOfThought" class ApproachConfig(BaseModel, ABC): max_input_tokens: int = Field(default=3000, description="Maximum number of tokens in the input prompt.") - model: ModelConfigType = Field(default=DefaultModelConfig()) + model: ModelConfig = Field(default=llm_config.models.base_model_config) type: str = Field(..., description="The type of approach config") class Config: diff --git a/modules/text/module_text_llm/module_text_llm/basic_approach/generate_suggestions.py b/modules/text/module_text_llm/module_text_llm/basic_approach/generate_suggestions.py index 6cdf5b369..c4446d747 100644 --- a/modules/text/module_text_llm/module_text_llm/basic_approach/generate_suggestions.py +++ b/modules/text/module_text_llm/module_text_llm/basic_approach/generate_suggestions.py @@ -4,18 +4,17 @@ from athena.text import Exercise, Submission, Feedback from athena.logger import logger from llm_core.utils.llm_utils import ( - get_chat_prompt_with_formatting_instructions, + get_chat_prompt, check_prompt_length_and_omit_features_if_necessary, num_tokens_from_prompt, ) -from llm_core.utils.predict_and_parse import predict_and_parse +from llm_core.core.predict_and_parse import predict_and_parse from module_text_llm.config import BasicApproachConfig from module_text_llm.helpers.utils import add_sentence_numbers, get_index_range_from_line_range, format_grading_instructions from module_text_llm.basic_approach.prompt_generate_suggestions import AssessmentModel async def generate_suggestions(exercise: Exercise, submission: Submission, config: BasicApproachConfig, debug: bool) -> List[Feedback]: - model = config.model.get_model() # type: ignore[attr-defined] prompt_input = { "max_points": exercise.max_points, "bonus_points": exercise.bonus_points, @@ -25,11 +24,9 @@ async def generate_suggestions(exercise: Exercise, submission: Submission, confi "submission": add_sentence_numbers(submission.text) } - chat_prompt = get_chat_prompt_with_formatting_instructions( - model=model, + chat_prompt = get_chat_prompt( system_message=config.generate_suggestions_prompt.system_message, human_message=config.generate_suggestions_prompt.human_message, - pydantic_object=AssessmentModel ) # Check if the prompt is too long and omit features if necessary (in order of importance) @@ -51,7 +48,7 @@ async def generate_suggestions(exercise: Exercise, submission: Submission, confi return [] result = await predict_and_parse( - model=model, + model=config.model, chat_prompt=chat_prompt, prompt_input=prompt_input, pydantic_object=AssessmentModel, @@ -59,7 +56,6 @@ async def generate_suggestions(exercise: Exercise, submission: Submission, confi f"exercise-{exercise.id}", f"submission-{submission.id}", ], - use_function_calling=True ) if debug: diff --git a/modules/text/module_text_llm/module_text_llm/chain_of_thought_approach/__init__.py b/modules/text/module_text_llm/module_text_llm/chain_of_thought_approach/__init__.py index b63b770e1..b2eee1e52 100644 --- a/modules/text/module_text_llm/module_text_llm/chain_of_thought_approach/__init__.py +++ b/modules/text/module_text_llm/module_text_llm/chain_of_thought_approach/__init__.py @@ -1,15 +1,18 @@ -from pydantic import BaseModel, Field +from llm_core.loaders.llm_config_loader import get_llm_config +from llm_core.models.model_config import ModelConfig +from pydantic import Field from typing import Literal -from llm_core.models import ModelConfigType, MiniModelConfig from module_text_llm.approach_config import ApproachConfig from module_text_llm.chain_of_thought_approach.prompt_generate_feedback import CoTGenerateSuggestionsPrompt from module_text_llm.chain_of_thought_approach.prompt_thinking import ThinkingPrompt +llm_config = get_llm_config() + class ChainOfThoughtConfig(ApproachConfig): # Defaults to the cheaper mini 4o model type: Literal['chain_of_thought'] = 'chain_of_thought' - model: ModelConfigType = Field(default=MiniModelConfig) # type: ignore + model: ModelConfig = Field(default=llm_config.models.base_model_config) thikning_prompt: ThinkingPrompt = Field(default=ThinkingPrompt()) generate_suggestions_prompt: CoTGenerateSuggestionsPrompt = Field(default=CoTGenerateSuggestionsPrompt()) \ No newline at end of file diff --git a/modules/text/module_text_llm/module_text_llm/chain_of_thought_approach/generate_suggestions.py b/modules/text/module_text_llm/module_text_llm/chain_of_thought_approach/generate_suggestions.py index 6b86dc978..4f160ae0b 100644 --- a/modules/text/module_text_llm/module_text_llm/chain_of_thought_approach/generate_suggestions.py +++ b/modules/text/module_text_llm/module_text_llm/chain_of_thought_approach/generate_suggestions.py @@ -1,5 +1,4 @@ -from typing import List, Optional -from pydantic import BaseModel, Field +from typing import List from athena import emit_meta from athena.text import Exercise, Submission, Feedback @@ -8,11 +7,11 @@ from module_text_llm.chain_of_thought_approach import ChainOfThoughtConfig from llm_core.utils.llm_utils import ( - get_chat_prompt_with_formatting_instructions, - check_prompt_length_and_omit_features_if_necessary, + check_prompt_length_and_omit_features_if_necessary, + get_chat_prompt, num_tokens_from_prompt, ) -from llm_core.utils.predict_and_parse import predict_and_parse +from llm_core.core.predict_and_parse import predict_and_parse from module_text_llm.helpers.utils import add_sentence_numbers, get_index_range_from_line_range, format_grading_instructions from module_text_llm.chain_of_thought_approach.prompt_thinking import InitialAssessmentModel @@ -20,7 +19,6 @@ async def generate_suggestions(exercise: Exercise, submission: Submission, config: ChainOfThoughtConfig, debug: bool) -> List[Feedback]: - model = config.model.get_model() # type: ignore[attr-defined] prompt_input = { "max_points": exercise.max_points, @@ -31,11 +29,9 @@ async def generate_suggestions(exercise: Exercise, submission: Submission, confi "submission": add_sentence_numbers(submission.text) } - chat_prompt = get_chat_prompt_with_formatting_instructions( - model=model, + chat_prompt = get_chat_prompt( system_message=config.thikning_prompt.system_message, human_message=config.thikning_prompt.human_message, - pydantic_object=InitialAssessmentModel ) @@ -59,15 +55,14 @@ async def generate_suggestions(exercise: Exercise, submission: Submission, confi return [] initial_result = await predict_and_parse( - model=model, + model=config.model, chat_prompt=chat_prompt, prompt_input=prompt_input, pydantic_object=InitialAssessmentModel, tags=[ f"exercise-{exercise.id}", f"submission-{submission.id}", - ], - use_function_calling=True + ] ) second_prompt_input = { @@ -76,24 +71,22 @@ async def generate_suggestions(exercise: Exercise, submission: Submission, confi } - second_chat_prompt = get_chat_prompt_with_formatting_instructions( - model=model, + second_chat_prompt = get_chat_prompt( system_message=config.generate_suggestions_prompt.second_system_message, human_message=config.generate_suggestions_prompt.answer_message, - pydantic_object=AssessmentModel) + ) result = await predict_and_parse( - model=model, + model=config.model, chat_prompt=second_chat_prompt, prompt_input=second_prompt_input, pydantic_object=AssessmentModel, tags=[ f"exercise-{exercise.id}", f"submission-{submission.id}", - ], - use_function_calling=True + ] ) - + if debug: emit_meta("generate_suggestions", { "prompt": chat_prompt.format(**prompt_input), diff --git a/modules/text/module_text_llm/module_text_llm/generate_evaluation.py b/modules/text/module_text_llm/module_text_llm/generate_evaluation.py index 50473dd3a..ff154ed85 100644 --- a/modules/text/module_text_llm/module_text_llm/generate_evaluation.py +++ b/modules/text/module_text_llm/module_text_llm/generate_evaluation.py @@ -1,16 +1,15 @@ from typing import List, Sequence, Dict, Literal +from llm_core.loaders.llm_config_loader import get_llm_config from pydantic import BaseModel, Field import json from athena.text import Exercise, Submission, Feedback from athena.logger import logger - -from llm_core.models import evaluation_model from llm_core.utils.llm_utils import ( - get_chat_prompt_with_formatting_instructions, + get_chat_prompt, check_prompt_length_and_omit_features_if_necessary, ) -from llm_core.utils.predict_and_parse import predict_and_parse +from llm_core.core.predict_and_parse import predict_and_parse from module_text_llm.helpers.utils import add_sentence_numbers, get_line_range_from_index_range from module_text_llm.prompts.generate_evaluation import system_message, human_message @@ -33,9 +32,6 @@ async def generate_evaluation( predicted_feedbacks: List[Feedback] ) -> Dict[int, dict]: - if evaluation_model is None: - raise EnvironmentError("No evaluation model available, please set up LLM_EVALUATION_MODEL correctly" - "by setting it to one of the available models logged during startup.") max_input_tokens = 3000 def feedback_to_dict(feedback: Feedback): @@ -56,11 +52,9 @@ def feedback_to_dict(feedback: Feedback): "predicted_feedbacks": json.dumps([feedback_to_dict(feedback) for feedback in predicted_feedbacks]), } - chat_prompt = get_chat_prompt_with_formatting_instructions( - model=evaluation_model, + chat_prompt = get_chat_prompt( system_message=system_message, human_message=human_message, - pydantic_object=Evaluation ) # Check if the prompt is too long and omit features if necessary (in order of importance) @@ -78,7 +72,7 @@ def feedback_to_dict(feedback: Feedback): return {} result = await predict_and_parse( - model=evaluation_model, + model=get_llm_config().models.base_model_config, chat_prompt=chat_prompt, prompt_input=prompt_input, pydantic_object=Evaluation, From b66c99f1de94eca5212d827bf8b4789d1c8a783a Mon Sep 17 00:00:00 2001 From: LeonWehrhahn Date: Fri, 3 Jan 2025 02:28:08 +0100 Subject: [PATCH 13/18] Refactor model configuration types to use ModelConfigType --- llm_core/README | 0 llm_core/llm_core/models/__init__.py | 6 ++++++ .../module_modeling_llm/module_modeling_llm/config.py | 9 +++++---- .../module_programming_llm/config.py | 4 ++-- .../module_text_llm/module_text_llm/approach_config.py | 4 ++-- .../chain_of_thought_approach/__init__.py | 4 ++-- 6 files changed, 17 insertions(+), 10 deletions(-) create mode 100644 llm_core/README diff --git a/llm_core/README b/llm_core/README new file mode 100644 index 000000000..e69de29bb diff --git a/llm_core/llm_core/models/__init__.py b/llm_core/llm_core/models/__init__.py index e69de29bb..6fb0b9f32 100644 --- a/llm_core/llm_core/models/__init__.py +++ b/llm_core/llm_core/models/__init__.py @@ -0,0 +1,6 @@ +from typing import Union + +from llm_core.models.openai_model_config import OpenAIModelConfig + + +ModelConfigType = Union[OpenAIModelConfig] \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/module_modeling_llm/config.py b/modules/modeling/module_modeling_llm/module_modeling_llm/config.py index b3998c7f4..48e084295 100644 --- a/modules/modeling/module_modeling_llm/module_modeling_llm/config.py +++ b/modules/modeling/module_modeling_llm/module_modeling_llm/config.py @@ -1,4 +1,5 @@ from llm_core.loaders.llm_config_loader import get_llm_config +from llm_core.models import ModelConfigType from pydantic import BaseModel, Field from athena import config_schema_provider @@ -47,10 +48,10 @@ class GenerateSuggestionsPrompt(BaseModel): class BasicApproachConfig(BaseModel): """This approach uses a LLM with a single prompt to generate feedback in a single step.""" max_input_tokens: int = Field(default=5000, description="Maximum number of tokens in the input prompt.") - generate_feedback: ModelConfig = Field(default=llm_config.models.base_model_config) - filter_feedback: ModelConfig = Field(default=llm_config.models.base_model_config) - review_feedback: ModelConfig = Field(default=llm_config.models.fast_reasoning_model_config) - generate_grading_instructions: ModelConfig = Field(default=llm_config.models.base_model_config) + generate_feedback: ModelConfigType = Field(default=llm_config.models.base_model_config) + filter_feedback: ModelConfigType = Field(default=llm_config.models.base_model_config) + review_feedback: ModelConfigType = Field(default=llm_config.models.fast_reasoning_model_config) + generate_grading_instructions: ModelConfigType = Field(default=llm_config.models.base_model_config) generate_suggestions_prompt: GenerateSuggestionsPrompt = Field(default=GenerateSuggestionsPrompt()) @config_schema_provider diff --git a/modules/programming/module_programming_llm/module_programming_llm/config.py b/modules/programming/module_programming_llm/module_programming_llm/config.py index f9fc60ef7..28f355b10 100644 --- a/modules/programming/module_programming_llm/module_programming_llm/config.py +++ b/modules/programming/module_programming_llm/module_programming_llm/config.py @@ -2,7 +2,7 @@ from llm_core.loaders.llm_config_loader import get_llm_config from pydantic import BaseModel, Field -from llm_core.models.model_config import ModelConfig +from llm_core.models import ModelConfigType from athena import config_schema_provider from module_programming_llm.prompts.generate_graded_suggestions_by_file import ( @@ -110,7 +110,7 @@ class BasicApproachConfig(BaseModel): """Defines a basic configuration for processing submissions, incorporating problem statement splitting, feedback generation, and file summarization.""" max_input_tokens: int = Field(default=3000, description="Maximum number of tokens in the input prompt.") - model: ModelConfig = Field(default=llm_config.models.base_model_config) + model: ModelConfigType = Field(default=llm_config.models.base_model_config) max_number_of_files: int = Field(default=25, description="Maximum number of files. If exceeded, it will prioritize the most important ones.") split_problem_statement_by_file_prompt: SplitProblemStatementsBasePrompt = Field(description="To be defined in " "subclasses.") generate_suggestions_by_file_prompt: SplitProblemStatementsBasePrompt = Field(description="To be defined in " "subclasses.") diff --git a/modules/text/module_text_llm/module_text_llm/approach_config.py b/modules/text/module_text_llm/module_text_llm/approach_config.py index 97cd1d32e..b614c707b 100644 --- a/modules/text/module_text_llm/module_text_llm/approach_config.py +++ b/modules/text/module_text_llm/module_text_llm/approach_config.py @@ -1,7 +1,7 @@ from abc import ABC from llm_core.loaders.llm_config_loader import get_llm_config from pydantic import BaseModel, Field -from llm_core.models.model_config import ModelConfig +from llm_core.models import ModelConfigType from enum import Enum llm_config = get_llm_config() @@ -12,7 +12,7 @@ class ApproachType(str, Enum): class ApproachConfig(BaseModel, ABC): max_input_tokens: int = Field(default=3000, description="Maximum number of tokens in the input prompt.") - model: ModelConfig = Field(default=llm_config.models.base_model_config) + model: ModelConfigType = Field(default=llm_config.models.base_model_config) type: str = Field(..., description="The type of approach config") class Config: diff --git a/modules/text/module_text_llm/module_text_llm/chain_of_thought_approach/__init__.py b/modules/text/module_text_llm/module_text_llm/chain_of_thought_approach/__init__.py index 7694973b7..fbc3a3e7e 100644 --- a/modules/text/module_text_llm/module_text_llm/chain_of_thought_approach/__init__.py +++ b/modules/text/module_text_llm/module_text_llm/chain_of_thought_approach/__init__.py @@ -1,5 +1,5 @@ from llm_core.loaders.llm_config_loader import get_llm_config -from llm_core.models.model_config import ModelConfig +from llm_core.models import ModelConfigType from pydantic import Field from typing import Literal @@ -11,7 +11,7 @@ class ChainOfThoughtConfig(ApproachConfig): type: Literal['chain_of_thought'] = 'chain_of_thought' - model: ModelConfig = Field(default=llm_config.models.base_model_config) + model: ModelConfigType = Field(default=llm_config.models.base_model_config) thikning_prompt: ThinkingPrompt = Field(default=ThinkingPrompt()) generate_suggestions_prompt: CoTGenerateSuggestionsPrompt = Field(default=CoTGenerateSuggestionsPrompt()) \ No newline at end of file From 8099d48a33ca7bf80b12dbf90a7c13938daa41e6 Mon Sep 17 00:00:00 2001 From: LeonWehrhahn Date: Fri, 3 Jan 2025 13:30:51 +0100 Subject: [PATCH 14/18] Add README for llm_core module; refactor OpenAI model config structure --- llm_core/README | 197 ++++++++++++++++++ llm_core/llm_core/models/__init__.py | 2 +- .../{ => providers}/openai_model_config.py | 4 +- .../models/{callbacks.py => usage_handler.py} | 0 llm_core/llm_core/utils/model_factory.py | 2 +- 5 files changed, 201 insertions(+), 4 deletions(-) rename llm_core/llm_core/models/{ => providers}/openai_model_config.py (98%) rename llm_core/llm_core/models/{callbacks.py => usage_handler.py} (100%) diff --git a/llm_core/README b/llm_core/README index e69de29bb..003be45a6 100644 --- a/llm_core/README +++ b/llm_core/README @@ -0,0 +1,197 @@ +## Overview + +The `llm_core` module provides a robust and extensible system for configuring and managing Large Language Models (LLMs) within the Athena framework. It allows for defining task-specific LLM models, configure their capabilities, and seamlessly integrate new LLM providers. This README provides a comprehensive guide to understanding and utilizing the `llm_core` module's features. + +## Key Features + +1. **Granular LLM Model Selection for Tasks:** + * Define different LLM models for distinct tasks within each module (e.g., `module_modeling_llm`, `module_programming_llm`). +2. **Flexible and Comprehensive LLM Model Configuration:** + * Support a diverse range of LLM models with varying capabilities by configuring model settings (e.g., `temperature`, `top_p`) and capability flags (e.g., `supports_function_calling`, `supports_structured_output`) through YAML files. +3. **Preserved Dynamic Configuration Overrides:** + * Retain the ability to dynamically override LLM configurations via `x-` headers in API requests (used in the Athena playground). +4. **Multi-Provider Support:** + * Easily add support for new LLM providers by extending the `ModelConfig` class. +5. **Version-Controlled Configuration:** + * LLM configurations are managed through YAML files under version control, ensuring consistent deployments and eliminating environment-specific discrepancies. + +## Core Components + +### 1. Configuration Files + +The `llm_core` module uses two types of YAML files to manage LLM configurations: + +* **`llm_capabilities.yml` (llm\_core level):** + * Defines the core capabilities and default settings for different LLM models. + * Specifies model-specific overrides to default settings. + * Resides at the top level of the `llm_core` directory, shared across all modules. + * Example: + + ```yaml + defaults: + temperature: 0.7 + top_p: 1.0 + supports_function_calling: true + supports_structured_output: true + + models: + openai_o1-mini: + max_completion_tokens: 8000 + temperature: 1.0 + top_p: 1.0 + n: 1 + presence_penalty: 0.0 + frequency_penalty: 0.0 + supports_system_messages: false + supports_function_calling: false + supports_structured_output: false + ``` + +* **`llm_config.yml` (Module-specific):** + * Defines the concrete LLM models to be used for different tasks within a specific module (e.g., `module_modeling_llm`). + * Located at the root level of each module. + * Example (in `module_modeling_llm/llm_config.yml`): + + ```yaml + models: + base_model: "azure_openai_gpt-4o" + mini_model: "openai_o1-mini" + ``` + +### 2. `ModelConfig` and Type System + +The `ModelConfig` class serves as an abstraction layer for different LLM providers. It defines a common interface for interacting with LLMs and explicitly declares their capabilities. + +* **`model_config.py` (llm\_core/models):** + * Contains the abstract `ModelConfig` class. + * Defines abstract methods like: + * `get_model()`: Returns an instance of the configured LLM. + * `supports_system_messages()`: Indicates if the model supports system messages. + * `supports_function_calling()`: Indicates if the model supports function calling. + * `supports_structured_output()`: Indicates if the model supports structured output. + +* **`providers/` (llm\_core/models/providers):** + * Contains provider-specific implementations of `ModelConfig`. + * **`openai_model_config.py`:** + * Implements `ModelConfig` for OpenAI models (including Azure OpenAI). + * `get_model()` constructs `ChatOpenAI` or `AzureChatOpenAI` objects based on YAML configurations. + * Implements capability flag methods based on the model's features. + * Uses the `get_model_capabilities` function from `llm_capabilities_loader.py` to merge default and model-specific settings. + * Adding new providers: + * Create a new subclass of `ModelConfig` in the `providers` directory. + * Implement the abstract methods to define the provider's capabilities and model instantiation logic. + +### 3. Loaders + +* **`llm_capabilities_loader.py` (llm\_core/loaders):** + * Loads the `llm_capabilities.yml` file. + * Provides the `get_model_capabilities(model_key)` function to retrieve the merged capabilities (defaults and overrides) for a given model. + +* **`llm_config_loader.py` (llm\_core/loaders):** + * Loads the module-specific `llm_config.yml` file. + * Uses the `create_config_for_model` function to create `ModelConfig` instances based on the model names specified in the YAML file. + * Caches the loaded `LLMConfig` to avoid reloading on subsequent calls. + * `get_llm_config(path: Optional[str] = None) -> LLMConfig` is the public function for accessing the cached `LLMConfig` instance. If not loaded yet, loads from YAML and materializes it. + +* **`openai_loader.py` (llm\_core/loaders):** + * Discovers available OpenAI and Azure OpenAI models using environment variables (`OPENAI_API_KEY`, `AZURE_OPENAI_API_KEY`, etc.). + * Creates an `OpenAIModel` Enum for referencing discovered models. + +### 4. Utils + +* **`model_factory.py` (llm\_core/utils):** + * Provides the `create_config_for_model(model_name)` function. + * Determines the provider (OpenAI, Azure, etc.) based on the `model_name` prefix. + * Creates the appropriate `ModelConfig` subclass instance (e.g., `OpenAIModelConfig`). + * Handles unknown providers by raising a `ValueError`. + +* **`append_format_instructions.py` (llm\_core/utils):** + * Appends format instructions to the chat prompts + +* **`llm_utils.py` (llm\_core/utils):** + * Provides utility functions for: + * Calculating the number of tokens in a string or prompt (`num_tokens_from_string`, `num_tokens_from_prompt`). + * Checking prompt length and omitting features if necessary (`check_prompt_length_and_omit_features_if_necessary`). + * Removing system messages if the model doesn't support them (`remove_system_message`). + +### 5. Core Logic + +* **`predict_and_parse.py` (llm\_core/core):** + * Provides the `predict_and_parse` function for making LLM predictions and parsing the output using a Pydantic model. + * Handles models with and without native structured output/function calling support. + * Adds appropriate tags to the LLM run based on the experiment environment. + +### 6. Callbacks + +* **`callbacks.py` (llm\_core/models):** + * Provides the `UsageHandler` callback to track and emit LLM usage metadata (input/output tokens, cost). + +## Usage + +### 1. Configuration + +1. **Define LLM Capabilities (`llm_capabilities.yml`):** + + * Specify default settings and capability flags for different LLM models. + * Add model-specific overrides as needed. +2. **Define Task-Specific Models (`llm_config.yml`):** + + * In each module directory (e.g., `module_modeling_llm`), create an `llm_config.yml` file. + * Specify the `base_model`, `mini_model`, `fast_reasoning_model`, and `long_reasoning_model` to use for different tasks within that module. +3. **Set Environment Variables:** + + * Define necessary environment variables for your LLM providers (e.g., `OPENAI_API_KEY`, `AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT`). + * Use the `.env.example` file as a template. + +### 2. Using LLMs in Modules + +1. **Import `get_llm_config`:** + + ```python + from llm_core.loaders.llm_config_loader import get_llm_config + ``` + +2. **Load the LLM Configuration:** + + ```python + llm_config = get_llm_config() # Loads the llm_config.yml from the current module's directory + ``` + +3. **Access `ModelConfig` Instances:** + + ```python + base_model_config = llm_config.models.base_model_config + mini_model_config = llm_config.models.mini_model_config + ``` + +4. **Use `ModelConfig` to Get LLM Instances and Check Capabilities:** + + ```python + base_model = base_model_config.get_model() + + if base_model_config.supports_function_calling(): + # Use function calling features + ``` + +## Extending with New Providers + +1. **Create a new `ModelConfig` subclass:** + + * In `llm_core/models/providers`, create a new Python file (e.g., `my_provider_model_config.py`). + * Define a new class that inherits from `llm_core.models.model_config.ModelConfig`. + * Implement the abstract methods: + * `get_model()`: Instantiate and return your provider's LLM object. + * `supports_system_messages()`: Return `True` if the provider supports system messages, `False` otherwise. + * `supports_function_calling()`: Return `True` if the provider supports function calling, `False` otherwise. + * `supports_structured_output()`: Return `True` if the provider supports structured output, `False` otherwise. +2. **Update `model_factory.py`:** + + * Add a new `elif` block to `find_provider_for_model` to recognize your provider's model name prefix. + * Add a corresponding `elif` block to `create_config_for_model` to instantiate your new `ModelConfig` subclass. +3. **Define Capabilities in `llm_capabilities.yml`:** + + * Add entries for your provider's models, specifying their default settings and capabilities. +4. **Use Your New Provider:** + + * In your module's `llm_config.yml`, specify the model name using your provider's prefix. + * The `llm_core` system will automatically use your new `ModelConfig` implementation. \ No newline at end of file diff --git a/llm_core/llm_core/models/__init__.py b/llm_core/llm_core/models/__init__.py index 6fb0b9f32..aed3796f0 100644 --- a/llm_core/llm_core/models/__init__.py +++ b/llm_core/llm_core/models/__init__.py @@ -1,6 +1,6 @@ from typing import Union -from llm_core.models.openai_model_config import OpenAIModelConfig +from llm_core.models.providers.openai_model_config import OpenAIModelConfig ModelConfigType = Union[OpenAIModelConfig] \ No newline at end of file diff --git a/llm_core/llm_core/models/openai_model_config.py b/llm_core/llm_core/models/providers/openai_model_config.py similarity index 98% rename from llm_core/llm_core/models/openai_model_config.py rename to llm_core/llm_core/models/providers/openai_model_config.py index 443942ffb..eb72a3f9b 100644 --- a/llm_core/llm_core/models/openai_model_config.py +++ b/llm_core/llm_core/models/providers/openai_model_config.py @@ -4,8 +4,8 @@ from langchain.base_language import BaseLanguageModel from llm_core.loaders.openai_loader import available_models -from .model_config import ModelConfig -from .callbacks import UsageHandler +from ..model_config import ModelConfig +from ..usage_handler import UsageHandler class OpenAIModelConfig(ModelConfig): """OpenAI LLM configuration.""" diff --git a/llm_core/llm_core/models/callbacks.py b/llm_core/llm_core/models/usage_handler.py similarity index 100% rename from llm_core/llm_core/models/callbacks.py rename to llm_core/llm_core/models/usage_handler.py diff --git a/llm_core/llm_core/utils/model_factory.py b/llm_core/llm_core/utils/model_factory.py index 097cbd06f..6d4c04ccb 100644 --- a/llm_core/llm_core/utils/model_factory.py +++ b/llm_core/llm_core/utils/model_factory.py @@ -1,6 +1,6 @@ from typing import Optional -from llm_core.models.openai_model_config import OpenAIModelConfig +from llm_core.models.providers.openai_model_config import OpenAIModelConfig def find_provider_for_model(model_name: str) -> Optional[str]: """ From 590e10136e12c80e467e306d9c3509429309d111 Mon Sep 17 00:00:00 2001 From: LeonWehrhahn Date: Fri, 3 Jan 2025 13:36:50 +0100 Subject: [PATCH 15/18] Enhance README for llm_core module with detailed content structure and usage guidelines --- llm_core/README | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/llm_core/README b/llm_core/README index 003be45a6..13b35e322 100644 --- a/llm_core/README +++ b/llm_core/README @@ -1,3 +1,19 @@ +## Content + +- [Overview](#overview) +- [Key Features](#key-features) +- [Core Components](#core-components) + - [1. Configuration Files](#1-configuration-files) + - [2. `ModelConfig` and Type System](#2-modelconfig-and-type-system) + - [3. Loaders](#3-loaders) + - [4. Utils](#4-utils) + - [5. Core Logic](#5-core-logic) + - [6. Callbacks](#6-callbacks) +- [Usage](#usage) + - [1. Configuration](#1-configuration) + - [2. Using LLMs in Modules](#2-using-llms-in-modules) +- [Extending with New Providers](#extending-with-new-providers) + ## Overview The `llm_core` module provides a robust and extensible system for configuring and managing Large Language Models (LLMs) within the Athena framework. It allows for defining task-specific LLM models, configure their capabilities, and seamlessly integrate new LLM providers. This README provides a comprehensive guide to understanding and utilizing the `llm_core` module's features. @@ -194,4 +210,5 @@ The `ModelConfig` class serves as an abstraction layer for different LLM provide 4. **Use Your New Provider:** * In your module's `llm_config.yml`, specify the model name using your provider's prefix. - * The `llm_core` system will automatically use your new `ModelConfig` implementation. \ No newline at end of file + * The `llm_core` system will automatically use your new `ModelConfig` implementation. + From 12f5428de30007b6995b589ca81ad8186eb714a3 Mon Sep 17 00:00:00 2001 From: LeonWehrhahn Date: Thu, 9 Jan 2025 21:42:08 +0100 Subject: [PATCH 16/18] Refactor model configs --- .../azure_loader.py} | 41 ++---- .../loaders/model_loaders/openai_loader.py | 36 +++++ .../models/providers/azure_model_config.py | 137 ++++++++++++++++++ .../models/providers/openai_model_config.py | 6 +- llm_core/llm_core/utils/model_factory.py | 10 +- llm_core/poetry.lock | 2 +- .../module_modeling_llm/llm_config.yml | 8 +- .../modeling/module_modeling_llm/poetry.lock | 13 +- 8 files changed, 208 insertions(+), 45 deletions(-) rename llm_core/llm_core/loaders/{openai_loader.py => model_loaders/azure_loader.py} (54%) create mode 100644 llm_core/llm_core/loaders/model_loaders/openai_loader.py create mode 100644 llm_core/llm_core/models/providers/azure_model_config.py diff --git a/llm_core/llm_core/loaders/openai_loader.py b/llm_core/llm_core/loaders/model_loaders/azure_loader.py similarity index 54% rename from llm_core/llm_core/loaders/openai_loader.py rename to llm_core/llm_core/loaders/model_loaders/azure_loader.py index f04a93ff7..42b2b20e3 100644 --- a/llm_core/llm_core/loaders/openai_loader.py +++ b/llm_core/llm_core/loaders/model_loaders/azure_loader.py @@ -1,35 +1,22 @@ +from enum import Enum import os -from llm_core.loaders.llm_capabilities_loader import get_model_capabilities -import openai import requests - from typing import Dict, List -from enum import Enum -from pydantic import Field, PrivateAttr, validator, PositiveInt + from langchain.base_language import BaseLanguageModel -from langchain_openai import AzureChatOpenAI, ChatOpenAI +from langchain_openai import AzureChatOpenAI from athena.logger import logger -# Discover available models from OpenAI / Azure -OPENAI_PREFIX = "openai_" +# Discover available models from Azure AZURE_OPENAI_PREFIX = "azure_openai_" -openai_available = bool(os.environ.get("OPENAI_API_KEY")) -azure_openai_available = bool(os.environ.get("AZURE_OPENAI_API_KEY")) - -available_models: Dict[str, BaseLanguageModel] = {} - -# OpenAI -if openai_available: - openai.api_type = "openai" - for model in openai.models.list(): - if "gpt" in model.id or "o1" in model.id: - available_models[OPENAI_PREFIX + model.id] = ChatOpenAI(model=model.id) +azure_available = bool(os.environ.get("AZURE_OPENAI_API_KEY")) +azure_available_models: Dict[str, BaseLanguageModel] = {} # Azure OpenAI -if azure_openai_available: +if azure_available: def _get_azure_openai_deployments() -> List[str]: base_url = f"{os.environ.get('AZURE_OPENAI_ENDPOINT')}/openai" headers = {"api-key": os.environ["AZURE_OPENAI_API_KEY"]} @@ -57,17 +44,17 @@ def _get_azure_openai_deployments() -> List[str]: ] for deployment in _get_azure_openai_deployments(): - available_models[AZURE_OPENAI_PREFIX + deployment] = AzureChatOpenAI(azure_deployment=deployment) + azure_available_models[AZURE_OPENAI_PREFIX + deployment] = AzureChatOpenAI(azure_deployment=deployment) -if available_models: - logger.info("Available openai models: %s", ", ".join(available_models.keys())) +if azure_available_models: + logger.info("Available azure models: %s", ", ".join(azure_available_models.keys())) else: - logger.warning("No openai/azure models discovered.") + logger.warning("No azure models discovered.") # Enum for referencing the discovered models -if available_models: - OpenAIModel = Enum('OpenAIModel', {name: name for name in available_models}) # type: ignore +if azure_available_models: + AzureModel = Enum('AzureModel', {name: name for name in azure_available_models}) # type: ignore else: - class OpenAIModel(str, Enum): + class AzureModel(str, Enum): pass \ No newline at end of file diff --git a/llm_core/llm_core/loaders/model_loaders/openai_loader.py b/llm_core/llm_core/loaders/model_loaders/openai_loader.py new file mode 100644 index 000000000..31802af4e --- /dev/null +++ b/llm_core/llm_core/loaders/model_loaders/openai_loader.py @@ -0,0 +1,36 @@ +from enum import Enum +import os +import openai +from typing import Dict + +from langchain.base_language import BaseLanguageModel +from langchain_openai import ChatOpenAI + +from athena.logger import logger + + +# Discover available models from OpenAI +OPENAI_PREFIX = "openai_" +openai_available = bool(os.environ.get("OPENAI_API_KEY")) + +openai_available_models: Dict[str, BaseLanguageModel] = {} + +# OpenAI +if openai_available: + openai.api_type = "openai" + for model in openai.models.list(): + if "gpt" in model.id or "o1" in model.id: + openai_available_models[OPENAI_PREFIX + model.id] = ChatOpenAI(model=model.id) + + +if openai_available_models: + logger.info("Available openai models: %s", ", ".join(openai_available_models.keys())) +else: + logger.warning("No openai models discovered.") + +# Enum for referencing the discovered models +if openai_available_models: + OpenAIModel = Enum('OpenAIModel', {name: name for name in openai_available_models}) # type: ignore +else: + class OpenAIModel(str, Enum): + pass \ No newline at end of file diff --git a/llm_core/llm_core/models/providers/azure_model_config.py b/llm_core/llm_core/models/providers/azure_model_config.py new file mode 100644 index 000000000..f0337749d --- /dev/null +++ b/llm_core/llm_core/models/providers/azure_model_config.py @@ -0,0 +1,137 @@ +from llm_core.loaders.model_loaders.openai_loader import OpenAIModel +from llm_core.loaders.llm_capabilities_loader import get_model_capabilities +from pydantic import Field, PrivateAttr, validator, PositiveInt +from langchain.base_language import BaseLanguageModel +from llm_core.loaders.model_loaders.azure_loader import AzureModel, azure_available_models + +from ..model_config import ModelConfig +from ..usage_handler import UsageHandler + +class AzureModelConfig(ModelConfig): + """OpenAI LLM configuration.""" + + model_name: AzureModel = Field( + description="The name of the model to use (e.g., openai_gpt-3.5-turbo)." + ) + + max_tokens: PositiveInt = Field( + 4000, + description="An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens." + ) + + temperature: float = Field( + default=0.0, ge=0, le=2, + description="""\ +What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, +while lower values like 0.2 will make it more focused and deterministic. + +We generally recommend altering this or `top_p` but not both.\ +""" + ) + + top_p: float = Field( + default=1, ge=0, le=1, + description="""\ +An alternative to sampling with temperature, called nucleus sampling, where the model considers the results +of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability +mass are considered. + +We generally recommend altering this or `temperature` but not both.\ +""" + ) + + presence_penalty: float = Field( + default=0, ge=-2, le=2, + description="""\ +Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, +increasing the model's likelihood to talk about new topics. + +[See more information about frequency and presence penalties.](https://platform.openai.com/docs/api-reference/parameter-details)\ +""" + ) + + frequency_penalty: float = Field( + default=0, ge=-2, le=2, + description="""\ +Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, +decreasing the model's likelihood to repeat the same line verbatim. + +[See more information about frequency and presence penalties.](https://platform.openai.com/docs/api-reference/parameter-details)\ +""" + ) + + # Private boolean flags + _supports_function_calling: bool = PrivateAttr(default=True) + _supports_structured_output: bool = PrivateAttr(default=True) + _supports_system_messages: bool = PrivateAttr(default=True) + + # Initialization: Merge YAML capabilities + def __init__(self, **data): + super().__init__(**data) + raw_key = self.model_name.value if hasattr(self.model_name, "value") else self.model_name + caps = get_model_capabilities(raw_key) + + fields_set_by_user = set(data.keys()) + + # If user didn't pass a certain field, override from YAML + if "temperature" not in fields_set_by_user and "temperature" in caps: + self.temperature = float(caps["temperature"]) + if "top_p" not in fields_set_by_user and "top_p" in caps: + self.top_p = float(caps["top_p"]) + if "presence_penalty" not in fields_set_by_user and "presence_penalty" in caps: + self.presence_penalty = float(caps["presence_penalty"]) + if "frequency_penalty" not in fields_set_by_user and "frequency_penalty" in caps: + self.frequency_penalty = float(caps["frequency_penalty"]) + if "max_tokens" not in fields_set_by_user and "max_tokens" in caps: + self.max_tokens = caps["max_tokens"] + + # set booleans from caps + self._supports_function_calling = bool(caps.get("supports_function_calling", True)) + self._supports_structured_output = bool(caps.get("supports_structured_output", True)) + self._supports_system_messages = bool(caps.get("supports_system_messages", True)) + + def supports_system_messages(self) -> bool: + return self._supports_system_messages + + def supports_function_calling(self) -> bool: + return self._supports_function_calling + + def supports_structured_output(self) -> bool: + return self._supports_structured_output + + @validator('max_tokens') + def max_tokens_must_be_positive(cls, v): + if v <= 0: + raise ValueError('max_tokens must be a positive integer') + return v + + def get_model(self) -> BaseLanguageModel: + """ + Create the ChatOpenAI/AzureChatOpenAI object. + Only add parameters that the model actually supports. + """ + model = azure_available_models[self.model_name.value] + kwargs = model.__dict__.copy() + + secrets = {secret: getattr(model, secret) for secret in model.lc_secrets.keys()} + kwargs.update(secrets) + + model_kwargs = kwargs.get("model_kwargs", {}) + + for attr, value in self.dict().items(): + if attr == "model_name": + continue + if hasattr(model, attr): + kwargs[attr] = value + else: + model_kwargs[attr] = value + + + kwargs["model_kwargs"] = model_kwargs + kwargs["callbacks"] = [UsageHandler()] + + new_model = model.__class__(**kwargs) + return new_model + + class Config: + title = 'Azure' \ No newline at end of file diff --git a/llm_core/llm_core/models/providers/openai_model_config.py b/llm_core/llm_core/models/providers/openai_model_config.py index eb72a3f9b..43d6f6372 100644 --- a/llm_core/llm_core/models/providers/openai_model_config.py +++ b/llm_core/llm_core/models/providers/openai_model_config.py @@ -1,8 +1,8 @@ -from llm_core.loaders.openai_loader import OpenAIModel +from llm_core.loaders.model_loaders.openai_loader import OpenAIModel from llm_core.loaders.llm_capabilities_loader import get_model_capabilities from pydantic import Field, PrivateAttr, validator, PositiveInt from langchain.base_language import BaseLanguageModel -from llm_core.loaders.openai_loader import available_models +from llm_core.loaders.model_loaders.openai_loader import openai_available_models from ..model_config import ModelConfig from ..usage_handler import UsageHandler @@ -110,7 +110,7 @@ def get_model(self) -> BaseLanguageModel: Create the ChatOpenAI/AzureChatOpenAI object. Only add parameters that the model actually supports. """ - model = available_models[self.model_name.value] + model = openai_available_models[self.model_name.value] kwargs = model.__dict__.copy() secrets = {secret: getattr(model, secret) for secret in model.lc_secrets.keys()} diff --git a/llm_core/llm_core/utils/model_factory.py b/llm_core/llm_core/utils/model_factory.py index 6d4c04ccb..ea6787efe 100644 --- a/llm_core/llm_core/utils/model_factory.py +++ b/llm_core/llm_core/utils/model_factory.py @@ -1,6 +1,8 @@ from typing import Optional +from llm_core.models.model_config import ModelConfig from llm_core.models.providers.openai_model_config import OpenAIModelConfig +from llm_core.models.providers.azure_model_config import AzureModelConfig def find_provider_for_model(model_name: str) -> Optional[str]: """ @@ -17,7 +19,7 @@ def find_provider_for_model(model_name: str) -> Optional[str]: else: return None -def create_config_for_model(model_name: str) -> OpenAIModelConfig: # or Union[OpenAIModelConfig, AzureOpenAIModelConfig, ...] +def create_config_for_model(model_name: str) -> ModelConfig: """ Create the appropriate ModelConfig subclass instance (OpenAI, Azure, etc.) depending on the provider indicated by `model_name`. @@ -26,8 +28,8 @@ def create_config_for_model(model_name: str) -> OpenAIModelConfig: # or Union[O if provider == "openai": return OpenAIModelConfig(model_name=model_name) elif provider == "azure_openai": - return OpenAIModelConfig(model_name=model_name) - elif provider == "replicate": - raise NotImplementedError("Replicate support is not implemented yet.") + return AzureModelConfig(model_name=model_name) + elif provider == "ollama": + raise NotImplementedError("ollama support is not implemented yet.") else: raise ValueError(f"Unknown provider for model name '{model_name}'") \ No newline at end of file diff --git a/llm_core/poetry.lock b/llm_core/poetry.lock index 38d29ad76..f26d5dd9c 100644 --- a/llm_core/poetry.lock +++ b/llm_core/poetry.lock @@ -2481,4 +2481,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.0" python-versions = "3.11.*" -content-hash = "f5b0912af9142a6bce7409aa8ee02b57242f7da97ed916d6cf9b98c06c8c331b" +content-hash = "c0f287638fd6bcfb443d162e213c1a08e2c31be3368a69d92202842a10d9f582" diff --git a/modules/modeling/module_modeling_llm/llm_config.yml b/modules/modeling/module_modeling_llm/llm_config.yml index 25842669a..738f70f42 100644 --- a/modules/modeling/module_modeling_llm/llm_config.yml +++ b/modules/modeling/module_modeling_llm/llm_config.yml @@ -1,5 +1,5 @@ models: - base_model: openai_gpt-4o - mini_model: openai_gpt-4o-mini - fast_reasoning_model: openai_o1-mini - long_reasoning_model: openai_o1-preview \ No newline at end of file + base_model: azure_openai_gpt-4o + mini_model: azure_openai_gpt-4o + fast_reasoning_model: azure_openai_gpt-4o + long_reasoning_model: azure_openai_gpt-4o \ No newline at end of file diff --git a/modules/modeling/module_modeling_llm/poetry.lock b/modules/modeling/module_modeling_llm/poetry.lock index ef71273fd..be2c5543f 100644 --- a/modules/modeling/module_modeling_llm/poetry.lock +++ b/modules/modeling/module_modeling_llm/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -996,7 +996,7 @@ langchain-community = "0.2.15" langchain-openai = "0.1.23" langsmith = ">=0.1.0,<0.2.0" nltk = "3.9.1" -openai = "1.51.1" +openai = "1.58.1" python-dotenv = "1.0.0" tiktoken = "0.7.0" @@ -1270,13 +1270,13 @@ files = [ [[package]] name = "openai" -version = "1.51.1" +version = "1.58.1" description = "The official Python library for the openai API" optional = false -python-versions = ">=3.7.1" +python-versions = ">=3.8" files = [ - {file = "openai-1.51.1-py3-none-any.whl", hash = "sha256:035ba637bef7523282b5b8d9f2f5fdc0bb5bc18d52af2bfc7f64e4a7b0a169fb"}, - {file = "openai-1.51.1.tar.gz", hash = "sha256:a4908d68e0a1f4bcb45cbaf273c5fbdc3a4fa6239bb75128b58b94f7d5411563"}, + {file = "openai-1.58.1-py3-none-any.whl", hash = "sha256:e2910b1170a6b7f88ef491ac3a42c387f08bd3db533411f7ee391d166571d63c"}, + {file = "openai-1.58.1.tar.gz", hash = "sha256:f5a035fd01e141fc743f4b0e02c41ca49be8fab0866d3b67f5f29b4f4d3c0973"}, ] [package.dependencies] @@ -1291,6 +1291,7 @@ typing-extensions = ">=4.11,<5" [package.extras] datalib = ["numpy (>=1)", "pandas (>=1.2.3)", "pandas-stubs (>=1.1.0.11)"] +realtime = ["websockets (>=13,<15)"] [[package]] name = "orjson" From e060cd1b5c04ba8470afb1b7404a8ccc9fa3ecb3 Mon Sep 17 00:00:00 2001 From: LeonWehrhahn Date: Fri, 31 Jan 2025 21:13:51 +0100 Subject: [PATCH 17/18] Add AzureModelConfig to ModelConfigType for multi-provider support --- llm_core/llm_core/models/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llm_core/llm_core/models/__init__.py b/llm_core/llm_core/models/__init__.py index aed3796f0..a546bf0b1 100644 --- a/llm_core/llm_core/models/__init__.py +++ b/llm_core/llm_core/models/__init__.py @@ -1,6 +1,7 @@ from typing import Union +from llm_core.models.providers.azure_model_config import AzureModelConfig from llm_core.models.providers.openai_model_config import OpenAIModelConfig -ModelConfigType = Union[OpenAIModelConfig] \ No newline at end of file +ModelConfigType = Union[OpenAIModelConfig, AzureModelConfig] \ No newline at end of file From 87a93b8df274da5123d5cbc7c503a5921b1e3628 Mon Sep 17 00:00:00 2001 From: LeonWehrhahn Date: Fri, 31 Jan 2025 21:57:50 +0100 Subject: [PATCH 18/18] Remove use_function_calling parameter from suggestion generation functions --- .../generate_graded_suggestions_by_file.py | 1 - .../generate_non_graded_suggestions_by_file.py | 1 - .../module_programming_llm/generate_summary_by_file.py | 1 - .../module_programming_llm/split_grading_instructions_by_file.py | 1 - .../module_programming_llm/split_problem_statement_by_file.py | 1 - 5 files changed, 5 deletions(-) diff --git a/modules/programming/module_programming_llm/module_programming_llm/generate_graded_suggestions_by_file.py b/modules/programming/module_programming_llm/module_programming_llm/generate_graded_suggestions_by_file.py index f39d626b6..0a6f905b2 100644 --- a/modules/programming/module_programming_llm/module_programming_llm/generate_graded_suggestions_by_file.py +++ b/modules/programming/module_programming_llm/module_programming_llm/generate_graded_suggestions_by_file.py @@ -265,7 +265,6 @@ async def generate_suggestions_by_file( chat_prompt=chat_prompt, prompt_input=prompt_input, pydantic_object=AssessmentModel, - use_function_calling=True, tags=[ f"exercise-{exercise.id}", f"submission-{submission.id}", diff --git a/modules/programming/module_programming_llm/module_programming_llm/generate_non_graded_suggestions_by_file.py b/modules/programming/module_programming_llm/module_programming_llm/generate_non_graded_suggestions_by_file.py index 715a90c5e..68a606a37 100644 --- a/modules/programming/module_programming_llm/module_programming_llm/generate_non_graded_suggestions_by_file.py +++ b/modules/programming/module_programming_llm/module_programming_llm/generate_non_graded_suggestions_by_file.py @@ -216,7 +216,6 @@ async def generate_suggestions_by_file( chat_prompt=chat_prompt, prompt_input=prompt_input, pydantic_object=ImprovementModel, - use_function_calling=True, tags=[ f"exercise-{exercise.id}", f"submission-{submission.id}", diff --git a/modules/programming/module_programming_llm/module_programming_llm/generate_summary_by_file.py b/modules/programming/module_programming_llm/module_programming_llm/generate_summary_by_file.py index c2b6592d2..f2a9c6fb7 100644 --- a/modules/programming/module_programming_llm/module_programming_llm/generate_summary_by_file.py +++ b/modules/programming/module_programming_llm/module_programming_llm/generate_summary_by_file.py @@ -118,7 +118,6 @@ async def generate_summary_by_file( chat_prompt=chat_prompt, prompt_input=prompt_input, pydantic_object=FileDescription, - use_function_calling=True, tags=[ f"exercise-{exercise.id}", f"submission-{submission.id}", diff --git a/modules/programming/module_programming_llm/module_programming_llm/split_grading_instructions_by_file.py b/modules/programming/module_programming_llm/module_programming_llm/split_grading_instructions_by_file.py index 32209366f..d27cd6860 100644 --- a/modules/programming/module_programming_llm/module_programming_llm/split_grading_instructions_by_file.py +++ b/modules/programming/module_programming_llm/module_programming_llm/split_grading_instructions_by_file.py @@ -96,7 +96,6 @@ async def split_grading_instructions_by_file( chat_prompt=chat_prompt, prompt_input=prompt_input, pydantic_object=SplitGradingInstructions, - use_function_calling=True, tags=[ f"exercise-{exercise.id}", f"submission-{submission.id}", diff --git a/modules/programming/module_programming_llm/module_programming_llm/split_problem_statement_by_file.py b/modules/programming/module_programming_llm/module_programming_llm/split_problem_statement_by_file.py index d000c5fb0..cb2e886cf 100644 --- a/modules/programming/module_programming_llm/module_programming_llm/split_problem_statement_by_file.py +++ b/modules/programming/module_programming_llm/module_programming_llm/split_problem_statement_by_file.py @@ -100,7 +100,6 @@ async def split_problem_statement_by_file( chat_prompt=chat_prompt, prompt_input=prompt_input, pydantic_object=SplitProblemStatement, - use_function_calling=True, tags=[ f"exercise-{exercise.id}", f"submission-{submission.id}",