From c9c9180ef66d9dca092be899b3f33c4cab708739 Mon Sep 17 00:00:00 2001
From: Tarek Mahmoud Sayed <tarekms@microsoft.com>
Date: Thu, 31 Oct 2024 15:41:50 -0700
Subject: [PATCH 1/3] Add the governance file cgmanifest.json for tokenizer's
 vocab files

---
 THIRD-PARTY-NOTICES.TXT                       |  4 +-
 cgmanifest.json                               | 49 +++++++++++++++++++
 .../Microsoft.ML.Tokenizers.Data.Gpt2.csproj  |  3 +-
 .../Model/TiktokenTokenizer.cs                |  2 +-
 .../TokenizerDataTests.cs                     |  2 +-
 .../TiktokenTests.cs                          |  2 +-
 6 files changed, 56 insertions(+), 6 deletions(-)
 create mode 100644 cgmanifest.json

diff --git a/THIRD-PARTY-NOTICES.TXT b/THIRD-PARTY-NOTICES.TXT
index 52364713d4..3bc1463084 100644
--- a/THIRD-PARTY-NOTICES.TXT
+++ b/THIRD-PARTY-NOTICES.TXT
@@ -86,8 +86,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 
-License notice for OpenAI Tiktoken Tokenizer
---------------------------------------------
+License notice for OpenAI Tiktoken Tokenizer & Tokenizer's vocab files
+----------------------------------------------------------------------
 
 https://github.com/openai/tiktoken/blob/main/LICENSE
 
diff --git a/cgmanifest.json b/cgmanifest.json
new file mode 100644
index 0000000000..4d2c818bd6
--- /dev/null
+++ b/cgmanifest.json
@@ -0,0 +1,49 @@
+{
+    "$schema": "https://json.schemastore.org/component-detection-manifest.json",
+    "Registrations": [
+      {
+        "Component": {
+          "Type": "other",
+          "other": {
+            "name": "cl100k_base.tiktoken",
+            "downloadUrl": "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
+            "hash": "sha1:6494e42d5aad2bbb441ea9793af9e7db335c8d9c"
+          }
+        },
+        "DevelopmentDependency": false
+      },
+      {
+        "Component": {
+          "Type": "other",
+          "other": {
+            "name": "o200k_base.tiktoken",
+            "downloadUrl": "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken",
+            "hash": "sha1:1d4fdeb17c52829ead47ac65e61197fd530b1c31"
+          }
+        },
+        "DevelopmentDependency": false
+      },
+      {
+        "Component": {
+          "Type": "other",
+          "other": {
+            "name": "p50k_base.tiktoken",
+            "downloadUrl": "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken",
+            "hash": "sha1:0ecf4ae6d454e7719bcf35f284eac0b73f37e3c9"
+          }
+        },
+        "DevelopmentDependency": false
+      },
+      {
+        "Component": {
+          "Type": "other",
+          "other": {
+            "name": "r50k_base.tiktoken",
+            "downloadUrl": "https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken",
+            "hash": "sha1:5674ba48e48e76284eb747c896a291dc5583c808"
+          }
+        },
+        "DevelopmentDependency": false
+      }
+    ]
+}
\ No newline at end of file
diff --git a/src/Microsoft.ML.Tokenizers.Data.Gpt2/Microsoft.ML.Tokenizers.Data.Gpt2.csproj b/src/Microsoft.ML.Tokenizers.Data.Gpt2/Microsoft.ML.Tokenizers.Data.Gpt2.csproj
index 15799111ee..66c89a06c1 100644
--- a/src/Microsoft.ML.Tokenizers.Data.Gpt2/Microsoft.ML.Tokenizers.Data.Gpt2.csproj
+++ b/src/Microsoft.ML.Tokenizers.Data.Gpt2/Microsoft.ML.Tokenizers.Data.Gpt2.csproj
@@ -11,10 +11,11 @@
     <!--
       The following file are compressed using the DeflateStream and embedded as resources in the assembly.
       The files are downloaded from the following sources and compressed to the Destination.
-        - gpt2.tiktoken:        https://fossies.org/linux/misc/whisper-20231117.tar.gz/whisper-20231117/whisper/assets/gpt2.tiktoken?m=b
+        - gpt2.tiktoken:        https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken
 
       The file under MIT copyright license https://github.com/openai/tiktoken/blob/main/LICENSE
 
+      Gpt2 vocab data is exact as the r50k_base vocab data, but with a different name.
       In the CompressFile task above we modify the file's content to elimenate the ranks, thus reducing the file size,
       since the rank corresponds to the line number in the file. For the file p50k_base.tiktoken,
       we introduce empty lines to replace any missing ranks, ensuring that the rank consistently aligns with the line number.
diff --git a/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs b/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs
index b169b2234f..67dec82979 100644
--- a/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs
+++ b/src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs
@@ -1153,7 +1153,7 @@ private static (Dictionary<string, int> SpecialTokens, Regex Regex, string Vocab
         private const string Cl100kBaseVocabFile = "cl100k_base.tiktoken.deflate";  // "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken"
         private const string P50RanksFile = "p50k_base.tiktoken.deflate";           // "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken"
         private const string R50RanksFile = "r50k_base.tiktoken.deflate";           // "https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken"
-        private const string GPT2File = "gpt2.tiktoken.deflate";                    // "https://fossies.org/linux/misc/legacy/whisper-20231117.tar.gz:b/whisper-20231117/whisper/assets/gpt2.tiktoken?m=b"
+        private const string GPT2File = "gpt2.tiktoken.deflate";                    // "https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken". Gpt2 is using the same encoding as R50kBase
         private const string O200kBaseFile = "o200k_base.tiktoken.deflate";         // "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken"
 
         internal const string Cl100kBaseEncodingName = "cl100k_base";
diff --git a/test/Microsoft.ML.Tokenizers.Data.Tests/TokenizerDataTests.cs b/test/Microsoft.ML.Tokenizers.Data.Tests/TokenizerDataTests.cs
index e165e931c9..dc1edd0ae2 100644
--- a/test/Microsoft.ML.Tokenizers.Data.Tests/TokenizerDataTests.cs
+++ b/test/Microsoft.ML.Tokenizers.Data.Tests/TokenizerDataTests.cs
@@ -34,8 +34,8 @@ public void TestMissingDataPackages(string modelName, string packageName)
 
         public static IEnumerable<object[]> ModelUrlData()
         {
+            // Gpt2 is covered by the r50k_base.tiktoken file
             yield return new object[] { @"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken" };
-            yield return new object[] { @"https://fossies.org/linux/misc/legacy/whisper-20231117.tar.gz:b/whisper-20231117/whisper/assets/gpt2.tiktoken?m=b" };
             yield return new object[] { @"https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken" };
             yield return new object[] { @"https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken" };
             yield return new object[] { @"https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken" };
diff --git a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs
index a8df1cc982..34f92647ae 100644
--- a/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs
+++ b/test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs
@@ -98,7 +98,7 @@ public async Task TestTokenizerCreation()
         public static IEnumerable<object[]> ModelUrlData()
         {
             yield return new object[] { GPT4, @"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken" };
-            yield return new object[] { GPT2, @"https://fossies.org/linux/misc/legacy/whisper-20231117.tar.gz:b/whisper-20231117/whisper/assets/gpt2.tiktoken?m=b" };
+            yield return new object[] { GPT2, @"https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken" }; // GPT2 uses the same encoding as R50kBase
             yield return new object[] { P50kBase, @"https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken" };
             yield return new object[] { R50kBase, @"https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken" };
             yield return new object[] { GPT4o, @"https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken" };

From 88b6ce3565a81dba707029dc17e38297d6c9075a Mon Sep 17 00:00:00 2001
From: Tarek Mahmoud Sayed <tarekms@microsoft.com>
Date: Thu, 31 Oct 2024 16:19:17 -0700
Subject: [PATCH 2/3] Address the feedback

---
 cgmanifest.json | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/cgmanifest.json b/cgmanifest.json
index 4d2c818bd6..2ab64d4591 100644
--- a/cgmanifest.json
+++ b/cgmanifest.json
@@ -3,47 +3,47 @@
     "Registrations": [
       {
         "Component": {
-          "Type": "other",
+          "type": "other",
           "other": {
             "name": "cl100k_base.tiktoken",
             "downloadUrl": "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
             "hash": "sha1:6494e42d5aad2bbb441ea9793af9e7db335c8d9c"
           }
         },
-        "DevelopmentDependency": false
+        "developmentDependency": false
       },
       {
         "Component": {
-          "Type": "other",
+          "type": "other",
           "other": {
             "name": "o200k_base.tiktoken",
             "downloadUrl": "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken",
             "hash": "sha1:1d4fdeb17c52829ead47ac65e61197fd530b1c31"
           }
         },
-        "DevelopmentDependency": false
+        "developmentDependency": false
       },
       {
         "Component": {
-          "Type": "other",
+          "type": "other",
           "other": {
             "name": "p50k_base.tiktoken",
             "downloadUrl": "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken",
             "hash": "sha1:0ecf4ae6d454e7719bcf35f284eac0b73f37e3c9"
           }
         },
-        "DevelopmentDependency": false
+        "developmentDependency": false
       },
       {
         "Component": {
-          "Type": "other",
+          "type": "other",
           "other": {
             "name": "r50k_base.tiktoken",
             "downloadUrl": "https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken",
             "hash": "sha1:5674ba48e48e76284eb747c896a291dc5583c808"
           }
         },
-        "DevelopmentDependency": false
+        "developmentDependency": false
       }
     ]
 }
\ No newline at end of file

From c2171cd4108d7c035bc18877a427ba56dd0135cc Mon Sep 17 00:00:00 2001
From: Tarek Mahmoud Sayed <tarekms@microsoft.com>
Date: Fri, 1 Nov 2024 09:23:15 -0700
Subject: [PATCH 3/3] apply more schema requirements on the doc

---
 cgmanifest.json | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/cgmanifest.json b/cgmanifest.json
index 2ab64d4591..3dc90f550f 100644
--- a/cgmanifest.json
+++ b/cgmanifest.json
@@ -1,11 +1,13 @@
 {
     "$schema": "https://json.schemastore.org/component-detection-manifest.json",
-    "Registrations": [
+    "version": 1,
+    "registrations": [
       {
-        "Component": {
+        "component": {
           "type": "other",
           "other": {
             "name": "cl100k_base.tiktoken",
+            "version": "1",
             "downloadUrl": "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
             "hash": "sha1:6494e42d5aad2bbb441ea9793af9e7db335c8d9c"
           }
@@ -13,10 +15,11 @@
         "developmentDependency": false
       },
       {
-        "Component": {
+        "component": {
           "type": "other",
           "other": {
             "name": "o200k_base.tiktoken",
+            "version": "1",
             "downloadUrl": "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken",
             "hash": "sha1:1d4fdeb17c52829ead47ac65e61197fd530b1c31"
           }
@@ -24,10 +27,11 @@
         "developmentDependency": false
       },
       {
-        "Component": {
+        "component": {
           "type": "other",
           "other": {
             "name": "p50k_base.tiktoken",
+            "version": "1",
             "downloadUrl": "https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken",
             "hash": "sha1:0ecf4ae6d454e7719bcf35f284eac0b73f37e3c9"
           }
@@ -35,10 +39,11 @@
         "developmentDependency": false
       },
       {
-        "Component": {
+        "component": {
           "type": "other",
           "other": {
             "name": "r50k_base.tiktoken",
+            "version": "1",
             "downloadUrl": "https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken",
             "hash": "sha1:5674ba48e48e76284eb747c896a291dc5583c808"
           }