diff --git a/app/pipeline/inconsistency_check_pipeline.py b/app/pipeline/inconsistency_check_pipeline.py
index d7f08045..3ac108fb 100644
--- a/app/pipeline/inconsistency_check_pipeline.py
+++ b/app/pipeline/inconsistency_check_pipeline.py
@@ -1,9 +1,10 @@
import logging
-from typing import Optional
+import re
+
+from typing import Dict, Optional
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.prompts import PromptTemplate
from langchain_core.runnables import Runnable
+from langchain_core.prompts import PromptTemplate
from langsmith import traceable
from app.common.PipelineEnum import PipelineEnum
@@ -12,19 +13,25 @@
from app.llm.langchain.iris_langchain_chat_model import IrisLangchainChatModel
from app.pipeline import Pipeline
from app.web.status.status_update import InconsistencyCheckCallback
-from app.pipeline.prompts.inconsistency_check_prompts import basic_prompt
+from app.pipeline.prompts.inconsistency_check_prompts import (
+ solver_prompt,
+ prettify_prompt,
+)
logger = logging.getLogger(__name__)
class InconsistencyCheckPipeline(Pipeline):
- pipeline: Runnable
llm: IrisLangchainChatModel
callback: InconsistencyCheckCallback
+ solver: Runnable
+ prettify: Runnable
+
def __init__(self, callback: Optional[InconsistencyCheckCallback] = None):
super().__init__(implementation_id="inconsistency_check_pipeline")
completion_args = CompletionArguments(temperature=0, max_tokens=2000)
+
self.llm = IrisLangchainChatModel(
request_handler=CapabilityRequestHandler(
requirements=RequirementList(
@@ -34,8 +41,12 @@ def __init__(self, callback: Optional[InconsistencyCheckCallback] = None):
),
completion_args=completion_args,
)
- self.prompt = PromptTemplate.from_template(basic_prompt)
- self.pipeline = self.prompt | self.llm | StrOutputParser()
+ self.solver_prompt = PromptTemplate.from_template(solver_prompt)
+ self.solver = self.solver_prompt | self.llm
+
+ self.prettify_prompt = PromptTemplate.from_template(prettify_prompt)
+ self.prettify = self.prettify_prompt | self.llm
+
self.callback = callback
self.tokens = []
@@ -54,18 +65,56 @@ def __call__(self, dto: InconsistencyCheckPipelineExecutionDTO, **kwargs):
logger.info("Running inconsistency check pipeline...")
self.callback.in_progress()
- template_repository = "\n".join(
- f"\n{file_content}"
- for file_path, file_content in dto.exercise.template_repository.items()
+ # First, for each file in the exercise, we will check for consistency issues via the solver pipeline
+ consistency_issues: Dict[str, str] = {}
+ file_paths = set(dto.exercise.template_repository.keys()) | set(
+ dto.exercise.solution_repository.keys()
)
+ solver_inputs = [
+ {
+ "file_path": file_path,
+ "problem_statement": dto.exercise.problem_statement,
+ "template_file": dto.exercise.template_repository.get(
+ file_path, "no file found"
+ ),
+ "solution_file": dto.exercise.solution_repository.get(
+ file_path, "no file found"
+ ),
+ }
+ for file_path in file_paths
+ ]
+ file_responses = self.solver.map().invoke(solver_inputs)
+ consistency_issues = {
+ file_path: response.content
+ for file_path, response in zip(file_paths, file_responses)
+ }
- response: str = self.pipeline.invoke(
+ # Second, we will prettify the consistency issues and provide a summary using the prettify pipeline
+ formatted_consistency_issues = "\n".join(
+ [
+ f"\n{issues}\n"
+ for file_path, issues in consistency_issues.items()
+ ]
+ )
+ summary_response = self.prettify.invoke(
{
"problem_statement": dto.exercise.problem_statement,
- "template_repository": template_repository,
+ "consistency_issues": formatted_consistency_issues,
}
)
- self._append_tokens(self.llm.tokens, PipelineEnum.IRIS_INCONSISTENCY_CHECK)
+ result = summary_response.content.strip()
- self.callback.done(final_result=response, tokens=self.tokens)
+ # Remove ``` from start and end if exists
+ if result.startswith("```") and result.endswith("```"):
+ result = result[3:-3]
+ if result.startswith("markdown"):
+ result = result[8:]
+ result = result.strip()
+
+ # Remove first heading or heading containing 'Summary of Consistency Issues'
+ result = re.sub(r"^#\s.*?\n", "", result)
+ result = re.sub(r"^#+.*?Summary of Consistency Issues\s*\n", "", result)
+
+ self._append_tokens(self.llm.tokens, PipelineEnum.IRIS_INCONSISTENCY_CHECK)
+ self.callback.done(final_result=result, tokens=self.tokens)
diff --git a/app/pipeline/prompts/inconsistency_check_prompts.py b/app/pipeline/prompts/inconsistency_check_prompts.py
index 02242f60..47c898dc 100644
--- a/app/pipeline/prompts/inconsistency_check_prompts.py
+++ b/app/pipeline/prompts/inconsistency_check_prompts.py
@@ -1,34 +1,63 @@
-basic_prompt = """\
+solver_prompt = """\
-As detail-oriented expert, find inconsistencies between the provided problem statement and the template repository of \
-a programming exercise.
-The student will use the the template repository to write code that solves the problem statement.
-
-Checks:
-- Given the problem statement, identify any missing or incorrect information in the template repository.
-- Given the template repository, identify any missing or incorrect information in the problem statement.
-- Ensure that the theme of the problem statement is consistent with the template repository.
-- Ensure that the problem statement is clear and concise and it covers everything that the student needs to know in \
-order to solve the exercise.
-
-It is not an inconsistency, if the problem statement clearly states that the student is responsible for writing a \
-specific part of the code.
+You are a detail-oriented expert instructor at an Ivy League university ensuring the quality of programming exercises. \
+Your task is to find consistency issues as part of the exercise creation process to make sure that the exercise is \
+without any errors or inconsistencies that might confuse students. Your teaching assistants will use your feedback to \
+improve the exercise.
+
+Parts of a programming exercise:
+ - Problem statement: The description of the exercise containing tasks that the student needs to solve.
+ - Template repository: The starting point from which the student will start solving the exercise.
+ - Solution repository: The sample solution set by the instructor to compare the student's solution against.
+
+To not overburden you, you will be provided with the problem statement and one of the template plus solution files \
+at a time. You need to compare the problem statement with the template file and identify any consistency issues.
+
+
+
+{problem_statement}
+
+
+
+{template_file}
+
+
+
+{solution_file}
+
+
+
+Respond with any potential consistency issues found in the exercise formatted in markdown. \
+Just provide the easily digestible formatted markdown without other explanations. It is fine to provide no issues if \
+you are confident that the files are consistent.
+
+"""
+
+prettify_prompt = """\
+
+You are a detail-oriented expert instructor at an Ivy League university ensuring the quality of programming exercises. \
+Your task is to find consistency issues as part of the exercise creation process to make sure that the exercise is \
+without any errors or inconsistencies that might confuse students.
+In a previous step you already found potential consistency issues as part of the exercise creation process on a file \
+level. Now, you need to summarize the issues found in the exercise so the teaching assistants can fix them.
+
+Parts of a programming exercise:
+ - Problem statement: The description of the exercise containing tasks that the student needs to solve.
+ - Template repository: The starting point from which the student will start solving the exercise.
+ - Solution repository: The sample solution set by the instructor to compare the student's solution against.
-
+
{problem_statement}
-
+
-
-{template_repository}
-
+
+{consistency_issues}
+
-Be smart about it, give a structured and actionable response that an instructor can use to significantly improve the \
-exercise. Clearly state where the inconsistency lies. Do not make up inconsistencies just to have something to say.
-It needs to be very comprehensive and detailed, imagine some inconsistencies slipped through, students in the exam \
-will be confused and frustrated. This is a high stakes exam, so we need to be very thorough.
-You will be legally responsible for the quality of the exercise, so make sure you do the absolute best job possible, \
-otherwise you will be held accountable in the court of law. Do not quote whole files! 🔫
+Respond with a summary of the consistency issues found in the exercise, stay specific and clear so the issues can be \
+easily fixed by the teaching assistants. Make it clear which file path contains the issues. Just provide the easily \
+digestible formatted markdown without other explanations.
"""