strip leading whitespace

karpathy · Jul 27, 2023 · e5752e1 · e5752e1
1 parent 25b50ee
commit e5752e1
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/run.c b/run.c
@@ -467,7 +467,9 @@ int main(int argc, char *argv[]) {
             // we now want to sample from this distribution to get the next token
             next = sample(state.logits, config.vocab_size);
         }
-        printf("%s", vocab[next]);
+        // following BOS token (1), sentencepiece decoder strips any leading whitespace (see PR #89)
+        char *token_str = (token == 1 && vocab[next][0] == ' ') ? vocab[next]+1 : vocab[next];
+        printf("%s", token_str);
         fflush(stdout);
 
         // advance forward