Merge pull request #86 from cvs-health/release-branch/v0.3.1

Release PR: v0.3.1

CONTRIBUTING.md

@@ -47,7 +47,7 @@ Please ensure your pull request adheres to the following guidelines:
 
 ## Development Setup
 
-1. Clone the repository: `git clone https://github.aetna.com/cvs-health/langfair`
+1. Clone the repository: `git clone https://github.com/cvs-health/langfair.git`
 2. Navigate to the project directory: `cd langfair`
 3. Create and activate a virtual environment (using `venv` or `conda`)
 4. Install dependencies: `poetry install`

README.md

@@ -10,7 +10,7 @@
 [![](https://img.shields.io/badge/arXiv-2407.10853-B31B1B.svg)](https://arxiv.org/abs/2407.10853)
 
 
-LangFair is a comprehensive Python library designed for conducting bias and fairness assessments of large language model (LLM) use cases. This repository includes a comprehensive framework for [choosing bias and fairness metrics](https://github.com/cvs-health/langfair/tree/main#choosing-bias-and-fairness-metrics-for-an-llm-use-case), along with [demo notebooks](https://github.com/cvs-health/langfair/tree/main/examples) and a [technical playbook](https://arxiv.org/abs/2407.10853) that discusses LLM bias and fairness risks, evaluation metrics, and best practices. 
+LangFair is a comprehensive Python library designed for conducting bias and fairness assessments of large language model (LLM) use cases. This repository includes a comprehensive framework for [choosing bias and fairness metrics](https://github.com/cvs-health/langfair/tree/main#-choosing-bias-and-fairness-metrics-for-an-llm-use-case), along with [demo notebooks](https://github.com/cvs-health/langfair/tree/main/examples) and a [technical playbook](https://arxiv.org/abs/2407.10853) that discusses LLM bias and fairness risks, evaluation metrics, and best practices. 
 
 Explore our [documentation site](https://cvs-health.github.io/langfair/) for detailed instructions on using LangFair.
 

examples/evaluations/classification/classification_metrics_demo.ipynb

@@ -78,7 +78,7 @@
    "source": [
     "#### Classification Metrics\n",
     "***\n",
-    "##### `ClassificationMetrics()` - For calculating FaiRLLM (Fairness of Recommendation via LLM) metrics (class)\n",
+    "##### `ClassificationMetrics()` - Pairwise classification fairness metrics (class)\n",
     "\n",
     "**Class parameters:**\n",
     "- `metric_type` - (**{'all', 'assistive', 'punitive', 'representation'}, default='all'**) Specifies which metrics to use.\n",

langfair/generator/counterfactual.py

@@ -191,12 +191,7 @@ def parse_texts(
             List of length `len(texts)` with each element being a list of identified protected
             attribute words in provided text
         """
-        assert not (custom_list and attribute), """
-        Either custom_list or attribute must be None.
-        """
-        assert custom_list or attribute in ["race", "gender"], """
-        If custom_list is None, attribute must be 'race' or 'gender'.
-        """
+        self._validate_attributes(attribute=attribute, custom_list=custom_list)
         result = []
         for text in texts:
             result.append(
@@ -234,13 +229,9 @@ def create_prompts(
         dict
             Dictionary containing counterfactual prompts
         """
-        assert not (custom_dict and attribute), """
-        Either custom_dict or attribute must be None.
-        """
-        assert custom_dict or attribute in [
-            "gender",
-            "race",
-        ], "If custom_dict is None, attribute must be 'gender' or 'race'."
+        self._validate_attributes(
+            attribute=attribute, custom_dict=custom_dict, for_parsing=False
+        )
 
         custom_list = (
             list(itertools.chain(*custom_dict.values())) if custom_dict else None
@@ -412,13 +403,94 @@ async def generate_responses(
             },
         }
 
+    def check_ftu(
+        self,
+        prompts: List[str],
+        attribute: Optional[str] = None,
+        custom_list: Optional[List[str]] = None,
+        subset_prompts: bool = True,
+    ) -> Dict[str, Any]:
+        """
+        Checks for fairness through unawarenss (FTU) based on a list of prompts and a specified protected
+        attribute
+
+        Parameters
+        ----------
+        prompts : list of strings
+            A list of prompts to be parsed for protected attribute words
+
+        attribute : {'race','gender'}, default=None
+            Specifies what to parse for among race words and gender words. Must be specified
+            if custom_list is None
+
+        custom_list : List[str], default=None
+            Custom list of tokens to use for parsing prompts. Must be provided if attribute is None.
+
+        subset_prompts : bool, default=True
+            Indicates whether to return all prompts or only those containing attribute words
+
+        Returns
+        -------
+        dict
+            A dictionary with two keys: 'data' and 'metadata'.
+            'data' : dict
+                A dictionary containing the prompts and responses.
+                'prompt' : list
+                    A list of prompts.
+                'attribute_words' : list
+                    A list of attribute_words in each prompt.
+            'metadata' : dict
+                A dictionary containing metadata related to FTU.
+                'ftu_satisfied' : boolean
+                    Boolean indicator of whether or not prompts satisfy FTU
+                'filtered_prompt_count' : int
+                    The number of prompts that satisfy FTU.
+        """
+        self._validate_attributes(attribute=attribute, custom_list=custom_list)
+        attribute_to_print = (
+            "Protected attribute" if not attribute else attribute.capitalize()
+        )
+        attribute_words = self.parse_texts(
+            texts=prompts, attribute=attribute, custom_list=custom_list, 
+        )
+        prompts_subset = [
+            prompt for i, prompt in enumerate(prompts) if attribute_words[i]
+        ]
+        attribute_words_subset = [
+            aw for i, aw in enumerate(attribute_words) if attribute_words[i]
+        ]
+
+        n_prompts_with_attribute_words = len(prompts_subset)
+        ftu_satisfied = (n_prompts_with_attribute_words > 0)
+        ftu_text = " not " if ftu_satisfied else " "
+
+        ftu_print = (f"FTU is{ftu_text}satisfied.")
+        print(f"{attribute_to_print} words found in {len(prompts_subset)} prompts. {ftu_print}")
+
+        return {
+            "data": {
+                "prompt": prompts_subset if subset_prompts else prompts,
+                "attribute_words": attribute_words_subset if subset_prompts else attribute_words
+            },
+            "metadata": {
+                "ftu_satisfied": ftu_satisfied,
+                "n_prompts_with_attribute_words": n_prompts_with_attribute_words,
+                "attribute": attribute,
+                "custom_list": custom_list,
+                "subset_prompts": subset_prompts
+            }
+        }
+
     def _subset_prompts(
         self,
         prompts: List[str],
         attribute: Optional[str] = None,
         custom_list: Optional[List[str]] = None,
     ) -> Tuple[List[str], List[List[str]]]:
-        """Subset prompts that contain protected attribute words"""
+        """
+        Helper function to subset prompts that contain protected attribute words and also 
+        return the full set of parsing results
+        """
         attribute_to_print = (
             "Protected attribute" if not attribute else attribute.capitalize()
         )
@@ -498,9 +570,6 @@ def _sub_from_dict(
 
         return output_dict
 
-    ################################################################################
-    # Class for protected attribute scanning and replacing protected attribute words
-    ################################################################################
     @staticmethod
     def _get_race_subsequences(text: str) -> List[str]:
         """Used to check for string sequences"""
@@ -522,3 +591,29 @@ def _replace_race(text: str, target_race: str) -> str:
         for subseq in STRICT_RACE_WORDS:
             seq = seq.replace(subseq, race_replacement_mapping[subseq])
         return seq
+
+    @staticmethod
+    def _validate_attributes(
+        attribute: Optional[str] = None, 
+        custom_list: Optional[List[str]] = None, 
+        custom_dict: Optional[Dict[str, str]] = None,
+        for_parsing: bool = True 
+    ) -> None:
+        if for_parsing:
+            if (custom_list and attribute):
+                raise ValueError(
+                    "Either custom_list or attribute must be None."
+                ) 
+            if not (custom_list or attribute in ["race", "gender"]):
+                raise ValueError(
+                    "If custom_list is None, attribute must be 'race' or 'gender'."
+                ) 
+        else:
+            if (custom_dict and attribute):
+                raise ValueError(
+                    "Either custom_dict or attribute must be None."
+                ) 
+            if not (custom_dict or attribute in ["race", "gender"]):
+                raise ValueError(
+                    "If custom_dict is None, attribute must be 'race' or 'gender'."
+                )