From e175f14078d905aad7a2f245566eb5a418a2f3f3 Mon Sep 17 00:00:00 2001 From: Dylan Bouchard Date: Wed, 11 Dec 2024 21:26:59 +0000 Subject: [PATCH 1/6] update version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c1d9fc0..ed255a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langfair" -version = "0.2.0" +version = "0.2.1" description = "LangFair is a Python library for conducting use-case level LLM bias and fairness assessments" readme = "README.md" authors = ["Dylan Bouchard ", From 667bbc4945d710ec4d7fb5b0232794a6e3e82964 Mon Sep 17 00:00:00 2001 From: Dylan Bouchard Date: Wed, 11 Dec 2024 21:36:22 +0000 Subject: [PATCH 2/6] minor readme update --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0307113..3d6e016 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ pip install langfair ``` ### Usage Examples -Below are the code samples illustrating how to use LangFair to assess bias and fairness risks in text generation and summarization use cases. The below examples assumes the user has already defined a list of prompts from their use case `prompts`. +Below are code samples illustrating how to use LangFair to assess bias and fairness risks in text generation and summarization use cases. The below examples assumes the user has already defined a list of prompts from their use case `prompts`. ##### Generate LLM responses To generate responses, we can use LangFair's `ResponseGenerator` class. First, we must create a `langchain` LLM object. Below we use `ChatVertexAI`, but **any of [LangChain’s LLM classes](https://js.langchain.com/docs/integrations/chat/) may be used instead**. Note that `InMemoryRateLimiter` is to used to avoid rate limit errors. @@ -135,8 +135,8 @@ results # 'Stereotype': {'Stereotype Association': 0.42777777777777776, # 'Cooccurrence Bias': 0.37655962458699777, # 'Stereotype Fraction - gender': 0.08, -# 'Expected Maximum Stereotype - gender': 0.580355167388916, -# 'Stereotype Probability - gender': 1}, +# 'Expected Maximum Stereotype - gender': 0.80355167388916, +# 'Stereotype Probability - gender': 0.27036}, # 'Counterfactual': {'male-female': {'Cosine Similarity': 0.31671187, # 'RougeL Similarity': 0.2882948246689143, # 'Bleu Similarity': 0.13248873839336991, From 9025b4a173946801fcb1ef8870898b77962ba1d4 Mon Sep 17 00:00:00 2001 From: Dylan Bouchard Date: Wed, 11 Dec 2024 21:41:14 +0000 Subject: [PATCH 3/6] fix typo in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3d6e016..860ea88 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ pip install langfair ``` ### Usage Examples -Below are code samples illustrating how to use LangFair to assess bias and fairness risks in text generation and summarization use cases. The below examples assumes the user has already defined a list of prompts from their use case `prompts`. +Below are code samples illustrating how to use LangFair to assess bias and fairness risks in text generation and summarization use cases. The below examples assume the user has already defined a list of prompts from their use case, `prompts`. ##### Generate LLM responses To generate responses, we can use LangFair's `ResponseGenerator` class. First, we must create a `langchain` LLM object. Below we use `ChatVertexAI`, but **any of [LangChain’s LLM classes](https://js.langchain.com/docs/integrations/chat/) may be used instead**. Note that `InMemoryRateLimiter` is to used to avoid rate limit errors. From 0032d8d13f345e9802828c0be3474be5e13811e0 Mon Sep 17 00:00:00 2001 From: Dylan Bouchard Date: Wed, 11 Dec 2024 21:46:23 +0000 Subject: [PATCH 4/6] update examples --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 860ea88..479374f 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ stereo_result['metrics'] # # Output is below # {'Stereotype Association': 0.3172750176745329, # 'Cooccurrence Bias': 0.44766333654278373, -# 'Stereotype Fraction - gender': 0.15452} +# 'Stereotype Fraction - gender': 0.08} ``` ##### Generate counterfactual responses and compute metrics @@ -129,18 +129,18 @@ auto_object = AutoEval( results = await auto_object.evaluate() results # Output is below -# {'Toxicity': {'Toxic Fraction': 0.0, -# 'Expected Maximum Toxicity': 0.08870933699654415, -# 'Toxicity Probability': 0}, -# 'Stereotype': {'Stereotype Association': 0.42777777777777776, +# {'Toxicity': {'Toxic Fraction': 0.0004, +# 'Expected Maximum Toxicity': 0.013845130120171235, +# 'Toxicity Probability': 0.01}, +# 'Stereotype': {'Stereotype Association': 0.3172750176745329, # 'Cooccurrence Bias': 0.37655962458699777, # 'Stereotype Fraction - gender': 0.08, # 'Expected Maximum Stereotype - gender': 0.80355167388916, # 'Stereotype Probability - gender': 0.27036}, -# 'Counterfactual': {'male-female': {'Cosine Similarity': 0.31671187, -# 'RougeL Similarity': 0.2882948246689143, -# 'Bleu Similarity': 0.13248873839336991, -# 'Sentiment Bias': 0.0114}}} +# 'Counterfactual': {'male-female': {'Cosine Similarity': 0.8318708, +# 'RougeL Similarity': 0.5195852482361165, +# 'Bleu Similarity': 0.3278433712872481, +# 'Sentiment Bias': 0.0009947145187601957}}} ``` ## 📚 Example Notebooks From 0ae9064d188d3bc54aabbd887b1d67823d41716e Mon Sep 17 00:00:00 2001 From: Dylan Bouchard Date: Wed, 11 Dec 2024 21:48:59 +0000 Subject: [PATCH 5/6] fix typo --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 479374f..96d1bca 100644 --- a/README.md +++ b/README.md @@ -133,9 +133,9 @@ results # 'Expected Maximum Toxicity': 0.013845130120171235, # 'Toxicity Probability': 0.01}, # 'Stereotype': {'Stereotype Association': 0.3172750176745329, -# 'Cooccurrence Bias': 0.37655962458699777, +# 'Cooccurrence Bias': 0.44766333654278373, # 'Stereotype Fraction - gender': 0.08, -# 'Expected Maximum Stereotype - gender': 0.80355167388916, +# 'Expected Maximum Stereotype - gender': 0.60355167388916, # 'Stereotype Probability - gender': 0.27036}, # 'Counterfactual': {'male-female': {'Cosine Similarity': 0.8318708, # 'RougeL Similarity': 0.5195852482361165, From d1d22ed30317a7e48cac0eff91db94c4d133db74 Mon Sep 17 00:00:00 2001 From: Dylan Bouchard Date: Wed, 11 Dec 2024 21:49:53 +0000 Subject: [PATCH 6/6] fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 96d1bca..2c6ffb4 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ cf_result ``` ##### Alternative approach: Semi-automated evaluation with `AutoEval` -To streamline assessments for text generation and summarization use cases, the `AutoEval` class conducts that completes all of the aforementioned steps with two lines of code. +To streamline assessments for text generation and summarization use cases, the `AutoEval` class conducts a multi-step process that completes all of the aforementioned steps with two lines of code. ```python from langfair.auto import AutoEval auto_object = AutoEval(