From e6be60d72f837a6455ad79a29460d4ec5fc920ee Mon Sep 17 00:00:00 2001 From: Rajas Bansal Date: Mon, 17 Jul 2023 13:31:54 -0700 Subject: [PATCH 1/2] Add llm explanation to output dataframe --- docs/guide/accuracy/chain-of-thought.md | 12 +++++++----- src/autolabel/labeler.py | 5 ++++- src/autolabel/schema.py | 1 + src/autolabel/tasks/base.py | 2 ++ 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/guide/accuracy/chain-of-thought.md b/docs/guide/accuracy/chain-of-thought.md index e2205575..c64031ee 100644 --- a/docs/guide/accuracy/chain-of-thought.md +++ b/docs/guide/accuracy/chain-of-thought.md @@ -44,19 +44,20 @@ config = { "few_shot_selection": "semantic_similarity", "few_shot_num": 3, "example_template": "Context: {context}\nQuestion: {question}\nAnswer: Let's think step by step.\n{explanation}\n{answer}", - "chain_of_thought": true + "chain_of_thought": True } } ``` Notice the changes that we have made to the config compared to the config without Chain-of-Thought [here](/guide/tasks/question_answering_task): -* `chain_of_thought` flag - this tells labeling agent to expect an explanation for the answer, in the seed dataset as well as LLM generated responses. -* `explanation_column` - this is the column where the explanation for the seed examples will reside. -* `example_template` - Notice that the template contains contains the explanation column as well. This tells the config where the explanation should be put when using the seed examples. We use the `Let's think step by step` prompt to initiate the chain of thought in the model. -* `output_guidelines` - We are explicitly prompting the LLM to first output an explanation, and then the final answer. +- `chain_of_thought` flag - this tells labeling agent to expect an explanation for the answer, in the seed dataset as well as LLM generated responses. +- `explanation_column` - this is the column where the explanation for the seed examples will reside. +- `example_template` - Notice that the template contains contains the explanation column as well. This tells the config where the explanation should be put when using the seed examples. We use the `Let's think step by step` prompt to initiate the chain of thought in the model. +- `output_guidelines` - We are explicitly prompting the LLM to first output an explanation, and then the final answer. Now, in order to generate explanations for the seed examples, in case they were not manually generated is, + ```py from autolabel import LabelingAgent agent = LabelingAgent(config) @@ -68,6 +69,7 @@ Once these explanations are generated, the dataset looks like {{ read_csv('docs/assets/squad_with_explanation_preview.csv') }} Now to generate labels for this dataset, all we have to do is, + ```py agent.plan('data/squad_v2_test.csv') agent.run('data/squad_v2_test.csv', max_items = 100) diff --git a/src/autolabel/labeler.py b/src/autolabel/labeler.py index e1091830..a3fe37f1 100644 --- a/src/autolabel/labeler.py +++ b/src/autolabel/labeler.py @@ -256,7 +256,10 @@ def run( l.label for l in llm_labels ] if self.config.confidence(): - output_df["llm_confidence"] = [l.confidence_score for l in llm_labels] + output_df[self.config.task_name() + "_llm_confidence"] = [l.confidence_score for l in llm_labels] + + if self.config.chain_of_thought(): + output_df[self.config.task_name() + '_llm_explanation'] = [l.explanation for l in llm_labels] # Only save to csv if output_name is provided or dataset is a string if not output_name and isinstance(dataset, str): diff --git a/src/autolabel/schema.py b/src/autolabel/schema.py index 57b10697..414bd4a0 100644 --- a/src/autolabel/schema.py +++ b/src/autolabel/schema.py @@ -80,6 +80,7 @@ class LLMAnnotation(BaseModel): confidence_score: Optional[float] = None generation_info: Optional[Dict[str, Any]] = None raw_response: Optional[str] = "" + explanation: Optional[str] = "" prompt: Optional[str] = "" diff --git a/src/autolabel/tasks/base.py b/src/autolabel/tasks/base.py index 6cf35f07..d7fd5139 100644 --- a/src/autolabel/tasks/base.py +++ b/src/autolabel/tasks/base.py @@ -69,6 +69,7 @@ def parse_llm_response( # This is done to handle the case where the model generates an explanation before generating the label if self.config.chain_of_thought(): try: + explanation = response.text.strip().split("\n")[0].strip() completion_text = extract_valid_json_substring( response.text.strip().split("\n")[-1].strip() ) @@ -106,4 +107,5 @@ def parse_llm_response( raw_response=response.text, prompt=prompt, curr_sample=json.dumps(curr_sample), + explanation = explanation if self.config.chain_of_thought() else "", ) From d38fddfaed88820aa8eb1647f09999332fd73a2e Mon Sep 17 00:00:00 2001 From: Rajas Bansal Date: Mon, 17 Jul 2023 13:48:19 -0700 Subject: [PATCH 2/2] formatting fixes --- src/autolabel/labeler.py | 10 +++++++--- src/autolabel/tasks/base.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/autolabel/labeler.py b/src/autolabel/labeler.py index a3fe37f1..5179b93f 100644 --- a/src/autolabel/labeler.py +++ b/src/autolabel/labeler.py @@ -256,10 +256,14 @@ def run( l.label for l in llm_labels ] if self.config.confidence(): - output_df[self.config.task_name() + "_llm_confidence"] = [l.confidence_score for l in llm_labels] - + output_df[self.config.task_name() + "_llm_confidence"] = [ + l.confidence_score for l in llm_labels + ] + if self.config.chain_of_thought(): - output_df[self.config.task_name() + '_llm_explanation'] = [l.explanation for l in llm_labels] + output_df[self.config.task_name() + "_llm_explanation"] = [ + l.explanation for l in llm_labels + ] # Only save to csv if output_name is provided or dataset is a string if not output_name and isinstance(dataset, str): diff --git a/src/autolabel/tasks/base.py b/src/autolabel/tasks/base.py index d7fd5139..2bf81f3b 100644 --- a/src/autolabel/tasks/base.py +++ b/src/autolabel/tasks/base.py @@ -107,5 +107,5 @@ def parse_llm_response( raw_response=response.text, prompt=prompt, curr_sample=json.dumps(curr_sample), - explanation = explanation if self.config.chain_of_thought() else "", + explanation=explanation if self.config.chain_of_thought() else "", )