diff --git a/src/Microsoft.ML.Tokenizers/Microsoft.ML.Tokenizers.csproj b/src/Microsoft.ML.Tokenizers/Microsoft.ML.Tokenizers.csproj index 78b6d7f49b..e50c62889b 100644 --- a/src/Microsoft.ML.Tokenizers/Microsoft.ML.Tokenizers.csproj +++ b/src/Microsoft.ML.Tokenizers/Microsoft.ML.Tokenizers.csproj @@ -2,11 +2,19 @@ - netstandard2.0 + netstandard2.0;net8.0 enable Microsoft.ML.Tokenizers contains the implmentation of the tokenization used in the NLP transforms. + + + + + + + + diff --git a/src/Microsoft.ML.Tokenizers/Model/BPE.cs b/src/Microsoft.ML.Tokenizers/Model/BPE.cs index 1537adec29..415e2e0e57 100644 --- a/src/Microsoft.ML.Tokenizers/Model/BPE.cs +++ b/src/Microsoft.ML.Tokenizers/Model/BPE.cs @@ -36,7 +36,7 @@ public string? UnknownToken if (value is null) { - if (VocabReverse.TryGetValue(0, out string v)) + if (VocabReverse.TryGetValue(0, out string? v)) { VocabReverse.Remove(0); if (Vocab.TryGetValue(v, out int id)) @@ -103,7 +103,7 @@ public Bpe(string vocabFile, string? mergesFile, string? unknownToken = null, st VocabReverse.Add(kvp.Value, kvp.Key); } - if (unknownToken is null && VocabReverse.TryGetValue(0, out string unkToken)) + if (unknownToken is null && VocabReverse.TryGetValue(0, out string? unkToken)) { unknownToken = unkToken; } @@ -187,7 +187,7 @@ public override IReadOnlyList Tokenize(string sequence) /// The mapped token of the Id. public override string? IdToToken(int id, bool skipSpecialTokens = false) { - if (VocabReverse.TryGetValue(id, out string value)) + if (VocabReverse.TryGetValue(id, out string? value)) { return value; } @@ -253,7 +253,7 @@ public override string[] Save(string path, string? prefix = null) } /// Read the given files to extract the vocab and merges - internal static (Dictionary?, Vec<(string, string)>) ReadFile(string? vocab, string? merges) + internal static (Dictionary?, Vec<(string, string)>) ReadFile(string vocab, string? merges) { Dictionary? dic; using (Stream stream = File.OpenRead(vocab)) @@ -320,7 +320,7 @@ internal static (Dictionary?, Vec<(string, string)>) ReadFile(strin [MethodImpl(MethodImplOptions.AggressiveInlining)] internal string CharToString(char c) { - if (_charToString.TryGetValue(c, out string v)) + if (_charToString.TryGetValue(c, out string? v)) { return v; } diff --git a/src/Microsoft.ML.Tokenizers/Model/BpeTrainer.cs b/src/Microsoft.ML.Tokenizers/Model/BpeTrainer.cs index 31942e17fb..fa65a8d540 100644 --- a/src/Microsoft.ML.Tokenizers/Model/BpeTrainer.cs +++ b/src/Microsoft.ML.Tokenizers/Model/BpeTrainer.cs @@ -83,7 +83,12 @@ public BpeTrainer( MinFrequency = minFrequency; VocabSize = vocabSize; Progress = progress; - SpecialTokens = new List(specialTokens); + + if (specialTokens is not null) + { + SpecialTokens = new List(specialTokens); + } + LimitAlphabet = limitAlphabet; InitialAlphabet = initialAlphabet; ContinuingSubwordPrefix = continuingSubwordPrefix; @@ -172,7 +177,7 @@ private void ComputeAlphabet(Dictionary wc, Dictionary [MethodImpl(MethodImplOptions.AggressiveInlining)] internal string CharToString(char c) { - if (_charToString.TryGetValue(c, out string v)) + if (_charToString.TryGetValue(c, out string? v)) { return v; } @@ -259,7 +264,7 @@ internal string CharToString(char c) // Then update counts int count = counts[i]; - if (!whereToUpdate.TryGetValue(curPair, out HashSet h)) + if (!whereToUpdate.TryGetValue(curPair, out HashSet? h)) { h = new HashSet(); whereToUpdate[curPair] = h; @@ -398,7 +403,7 @@ internal string CharToString(char c) if (change > 0) { - if (!whereToUpdate.TryGetValue(p, out HashSet h)) + if (!whereToUpdate.TryGetValue(p, out HashSet? h)) { h = new(); whereToUpdate[p] = h; diff --git a/src/Microsoft.ML.Tokenizers/Model/Cache.cs b/src/Microsoft.ML.Tokenizers/Model/Cache.cs index 2665bacfcf..269a580b1e 100644 --- a/src/Microsoft.ML.Tokenizers/Model/Cache.cs +++ b/src/Microsoft.ML.Tokenizers/Model/Cache.cs @@ -9,7 +9,7 @@ namespace Microsoft.ML.Tokenizers { - internal sealed class Cache + internal sealed class Cache where TKey : notnull { internal Cache() : this(Bpe.DefaultCacheCapacity) { } @@ -39,13 +39,13 @@ internal void Clear() internal List GetValues(IEnumerable keys) { - List? values = new(); + List values = new(); _cacheLock.EnterReadLock(); try { foreach (TKey key in keys) { - if (Map.TryGetValue(key, out TValue value)) + if (Map.TryGetValue(key, out TValue? value)) { values.Add(value); } @@ -61,7 +61,7 @@ internal List GetValues(IEnumerable keys) _cacheLock.EnterReadLock(); try { - if (Map.TryGetValue(key, out TValue value)) + if (Map.TryGetValue(key, out TValue? value)) { return value; } diff --git a/src/Microsoft.ML.Tokenizers/Model/EnglishRoberta.cs b/src/Microsoft.ML.Tokenizers/Model/EnglishRoberta.cs index edf0da3fcc..02fef91ded 100644 --- a/src/Microsoft.ML.Tokenizers/Model/EnglishRoberta.cs +++ b/src/Microsoft.ML.Tokenizers/Model/EnglishRoberta.cs @@ -429,7 +429,7 @@ private Dictionary GetVocabulary(Stream vocabularyStream) using StreamReader reader = new StreamReader(mergeStream); while (reader.Peek() >= 0) { - splitContents.Add(reader.ReadLine()); + splitContents.Add(reader.ReadLine()!); } } catch (Exception e) @@ -761,7 +761,11 @@ public void AddFromStream(Stream stream) while (reader.Peek() >= 0) { - string line = reader.ReadLine(); + string? line = reader.ReadLine(); + if (line is null) + { + continue; + } var splitLine = line.Trim().Split(' '); if (splitLine.Length != 2) diff --git a/src/Microsoft.ML.Tokenizers/Model/Model.cs b/src/Microsoft.ML.Tokenizers/Model/Model.cs index 81b9ab69f4..9f7fe9e698 100644 --- a/src/Microsoft.ML.Tokenizers/Model/Model.cs +++ b/src/Microsoft.ML.Tokenizers/Model/Model.cs @@ -20,6 +20,36 @@ public abstract class Model /// The list of tokens generated from the sequence tokenization. public abstract IReadOnlyList Tokenize(string sequence); + /// + /// Tokenize a split sequence string to a list of tokens. + /// + /// The text to tokenize. + /// Indicate if the token is a special token. + /// The list of tokens generated from the sequence tokenization. + public virtual IReadOnlyList Tokenize(string sequence, bool isSpecialToken) => Tokenize(sequence); + + /// + /// Tokenize a split sequence string to a list of Ids and add them to the accumulatedIds list. + /// + /// The sequence to split. + /// Indicate if the token is a special token. + /// The list of accumulated tokenized Ids. + /// True if the operation succeeded, false otherwise. + public virtual bool TokenizeToIds(string sequence, bool isSpecialToken, IList accumulatedIds) + { + if (accumulatedIds is null) + { + throw new ArgumentNullException(nameof(accumulatedIds)); + } + + var tokens = Tokenize(sequence); + foreach (var token in tokens) + { + accumulatedIds.Add(token.Id); + } + return true; + } + /// /// Map the token to tokenized Id. /// @@ -27,6 +57,14 @@ public abstract class Model /// The mapped Id of the token. public abstract int? TokenToId(string token); + /// + /// Map the token to tokenized id with the option to skip the special tokens. + /// + /// The token to map to Id + /// Indicate if want to skip the special tokens during the encoding. + /// The mapped Id of the token. + public virtual int? TokenToId(string token, bool skipSpecialTokens) => TokenToId(token); + /// /// Map the tokenized Id to the token. /// diff --git a/src/Microsoft.ML.Tokenizers/Model/Tiktoken.cs b/src/Microsoft.ML.Tokenizers/Model/Tiktoken.cs new file mode 100644 index 0000000000..cfc657afbc --- /dev/null +++ b/src/Microsoft.ML.Tokenizers/Model/Tiktoken.cs @@ -0,0 +1,414 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; + +namespace Microsoft.ML.Tokenizers +{ + /// + /// Represent the rapid Byte Pair Encoding model commonly referred to as Tiktoken. + /// + public sealed class Tiktoken : Model + { + private Dictionary _encoder = null!; + private IReadOnlyDictionary _decoder = null!; + private readonly LruCache _cache; + private IReadOnlyDictionary? _specialTokensEncoder; + private Dictionary? _specialTokensDecoder; + + private Dictionary _vocab = null!; + private static readonly List _emptyTokenList = new(); + + /// + /// Create a new Tiktoken tokenizer object. + /// + /// The path to the BPE rank file. + /// The dictionary mapping special tokens to Ids. + /// The size of the cache to use. + /// Thrown when is null or empty. + /// Thrown when failed to load the BPE rank file. + public Tiktoken(string tikTokenBpeFile, IReadOnlyDictionary? specialTokensEncoder = null, int cacheSize = LruCache.DefaultCacheSize) : this(cacheSize) + { + if (string.IsNullOrEmpty(tikTokenBpeFile)) + { + throw new ArgumentNullException(nameof(tikTokenBpeFile)); + } + + using (Stream stream = File.OpenRead(tikTokenBpeFile)) + { + Initialize(stream, specialTokensEncoder); + } + } + + /// + /// Create a new Tiktoken tokenizer object. + /// + /// The stream to the BPE rank file. + /// The dictionary mapping special tokens to Ids. + /// The size of the cache to use. + /// Thrown when is null or empty. + /// Thrown when failed to load the BPE rank file. + public Tiktoken(Stream tikTokenBpeFileStream, IReadOnlyDictionary? specialTokensEncoder = null, int cacheSize = LruCache.DefaultCacheSize) : this(cacheSize) + { + Initialize(tikTokenBpeFileStream, specialTokensEncoder); + } + + internal Tiktoken( + Dictionary encoder, + IReadOnlyDictionary decoder, + Dictionary vocab, + IReadOnlyDictionary? specialTokensEncoder = null, + int cacheSize = LruCache.DefaultCacheSize) : this(cacheSize) + { + Debug.Assert(encoder is not null); + Debug.Assert(decoder is not null); + Debug.Assert(vocab is not null); + + _encoder = encoder!; + _vocab = vocab!; + _decoder = decoder!; + + _specialTokensEncoder = specialTokensEncoder; + if (_specialTokensEncoder is not null) + { + _specialTokensDecoder = _specialTokensEncoder.ToDictionary(kvp => kvp.Value, kvp => kvp.Key); + } + } + + private Tiktoken(int cacheSize) + { + _cache = new LruCache(cacheSize); + } + + private void Initialize(Stream tikTokenBpeFileStream, IReadOnlyDictionary? specialTokensEncoder = null) + { + if (tikTokenBpeFileStream is null) + { + throw new ArgumentNullException(nameof(tikTokenBpeFileStream)); + } + + (_encoder, _vocab, _decoder) = LoadTikTokenBpe(tikTokenBpeFileStream); + + _specialTokensEncoder = specialTokensEncoder; + if (_specialTokensEncoder is not null) + { + _specialTokensDecoder = _specialTokensEncoder.ToDictionary(kvp => kvp.Value, kvp => kvp.Key); + } + } + + /// + /// Load BPE rank dictionary from a stream. + /// + /// Stream to the BPE rank file + /// Map of byte[] to integer token id + /// + internal static (Dictionary, Dictionary, IReadOnlyDictionary) LoadTikTokenBpe(Stream tikTokenBpeFileStream) + { + var encoder = new Dictionary(new ByteArrayComparer()); + var vocab = new Dictionary(); + var decoder = new Dictionary(); + + try + { + using (StreamReader reader = new StreamReader(tikTokenBpeFileStream)) + { + while (!reader.EndOfStream) + { + string? line = reader.ReadLine(); + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + int spaceIndex = line.IndexOf(' '); + if (spaceIndex <= 0 || spaceIndex >= line.Length - 1 || line.IndexOf(' ', spaceIndex + 1) >= 0) + { + throw new FormatException($"Invalid format in the BPE encoder file stream"); + } + + byte[] tokenBytes = Helpers.FromBase64String(line, 0, spaceIndex); + + if (Helpers.TryParseInt32(line, spaceIndex + 1, out int rank)) + { + encoder[tokenBytes] = rank; + decoder[rank] = tokenBytes; + + string decodedToken = Encoding.UTF8.GetString(tokenBytes); + + vocab[decodedToken] = rank; + } + else + { + throw new FormatException($"Can't parse {line.Substring(spaceIndex)} to integer"); + } + } + } + } + catch (Exception ex) + { + throw new InvalidOperationException($"Failed to load from BPE encoder file stream: {ex.Message}", ex); + } + + return (encoder, vocab, decoder); + } + + /// + /// Gets the dictionary mapping special tokens to Ids. + /// + /// The dictionary mapping special tokens to Ids. + public IReadOnlyDictionary? SpecialTokensEncoder => _specialTokensEncoder; + + /// + /// Tokenize a sequence string to a list of tokens. + /// + /// The sequence to tokenize. + /// The list of tokens generated from the sequence tokenization. + public override IReadOnlyList Tokenize(string sequence) => Tokenize(sequence, isSpecialToken: false); + + /// + /// Tokenize a split sequence string to a list of tokens. + /// + /// The text to tokenize. + /// Indicate if the token is a special token. + /// The list of tokens generated from the sequence tokenization. + public override IReadOnlyList Tokenize(string sequence, bool isSpecialToken) + { + List tokens; + + if (string.IsNullOrEmpty(sequence)) + { + return _emptyTokenList; + } + + if (isSpecialToken) + { + if (_specialTokensEncoder is null) + { + throw new InvalidOperationException($"The tokenizer doesn't have special tokens"); + } + + if (_specialTokensEncoder.TryGetValue(sequence, out int id)) + { + return new List { new(id, sequence, (0, sequence.Length)) }; + } + + throw new InvalidOperationException($"The special token {sequence} doesn't exist in the tokenizer"); + } + + if (_cache.Lookup(sequence, out int[] ids)) + { + tokens = new(ids.Length); + tokens.Add(new Token(ids[0], sequence, (0, sequence.Length))); + for (int i = 1; i < ids.Length; i++) + { + // One word split mapped to multiple Ids. Make the offset of the remaining token point at the end with zero width. + tokens.Add(new Token(ids[i], "", (sequence.Length, sequence.Length))); + } + + return tokens; + } + + // cache miss + if (_vocab.TryGetValue(sequence, out int mappedId)) + { + return new List { new(mappedId, sequence, (0, sequence.Length)) }; + } + + int[] encodedIds = BytePairEncoder.BytePairEncode(Encoding.UTF8.GetBytes(sequence), _encoder); + _cache.Add(sequence, encodedIds); + + tokens = new List(encodedIds.Length); + tokens.Add(new Token(encodedIds[0], sequence, (0, sequence.Length))); + for (int i = 1; i < encodedIds.Length; i++) + { + // One word split mapped to multiple Ids. Make the offset of the remaining token point at the end with zero width. + tokens.Add(new Token(encodedIds[i], "", (sequence.Length, sequence.Length))); + } + + return tokens; + } + + /// + /// Tokenize a split sequence string to a list of Ids. + /// + /// The sequence to tokenize. + /// Indicate if the token is a special token. + /// The list of accumulated Ids. + /// True if the operation succeeded, false otherwise. + public override bool TokenizeToIds(string sequence, bool isSpecialToken, IList accumulatedIds) + { + if (string.IsNullOrEmpty(sequence)) + { + return true; + } + + if (isSpecialToken) + { + if (_specialTokensEncoder is not null && _specialTokensEncoder.TryGetValue(sequence, out int id)) + { + accumulatedIds.Add(id); + return true; + } + + return false; + } + + if (_cache.Lookup(sequence, out int[] tokenIds)) + { + accumulatedIds.AddRange(tokenIds); + return true; + } + + if (_vocab.TryGetValue(sequence, out int mappedId)) + { + accumulatedIds.Add(mappedId); + return true; + } + + int[] encodedIds = BytePairEncoder.BytePairEncode(Encoding.UTF8.GetBytes(sequence), _encoder); + _cache.Add(sequence, encodedIds); + + accumulatedIds.AddRange(encodedIds); + return true; + } + + /// + /// Map the token to tokenized Id. + /// + /// The token to map to the Id. + /// The mapped Id of the token. + public override int? TokenToId(string token) => TokenToId(token, skipSpecialTokens: false); + + /// + /// Map the token to tokenized Id. + /// + /// The token to map to the Id. + /// Indicate if want to skip the special tokens during the encoding. + /// The mapped Id of the token. + public override int? TokenToId(string token, bool skipSpecialTokens) + { + if (string.IsNullOrEmpty(token)) + { + return 0; + } + + if (!skipSpecialTokens && _specialTokensEncoder is not null && _specialTokensEncoder.TryGetValue(token, out int specialTokenId)) + { + return specialTokenId; + } + + if (_cache.Lookup(token, out int[] ids)) + { + if (ids.Length == 1) + { + return ids[0]; + } + + return null; + } + + if (_vocab.TryGetValue(token, out int id)) + { + return id; + } + + int[] idsToCache = BytePairEncoder.BytePairEncode(Encoding.UTF8.GetBytes(token), _encoder); + _cache.Add(token, idsToCache); + + if (idsToCache.Length == 1) + { + return idsToCache[0]; + } + + return null; + } + + /// + /// Map the tokenized Id to the token. + /// + /// The Id to map to the token. + /// Indicate if want to skip the special tokens during the decoding. + /// The mapped token of the Id. + public override string? IdToToken(int id, bool skipSpecialTokens = false) + { + if (!skipSpecialTokens && _specialTokensDecoder is not null && _specialTokensDecoder.TryGetValue(id, out string? token)) + { + return token; + } + + if (_decoder.TryGetValue(id, out byte[]? tokenBytes)) + { + return Encoding.UTF8.GetString(tokenBytes); + } + + return null; + } + + internal string? IdsToString(IEnumerable ids, bool skipSpecialTokens = false) + { + if (ids is null) + { + return null; + } + + List utf8Bytes = new(); + bool useSpecialTokens = !skipSpecialTokens && _specialTokensDecoder is not null; + + foreach (int id in ids) + { + if (_decoder.TryGetValue(id, out byte[]? tokenBytes)) + { + utf8Bytes.AddRange(tokenBytes); + } + else if (useSpecialTokens && _specialTokensDecoder!.TryGetValue(id, out string? token)) + { + utf8Bytes.AddRange(Encoding.UTF8.GetBytes(token)); + } + else + { + return null; + } + } + + return utf8Bytes.Count > 0 ? Encoding.UTF8.GetString(utf8Bytes.ToArray()) : string.Empty; + } + + public override string? IdToString(int id, bool skipSpecialTokens = false) => IdToToken(id, skipSpecialTokens); + + /// + /// Gets the dictionary mapping tokens to Ids. + /// + public override IReadOnlyDictionary GetVocab() => _vocab; + + /// + /// Gets the dictionary size that map tokens to Ids. + /// + public override int GetVocabSize() => _vocab.Count; + + /// + /// Save the model data into the vocabulary and merges files. + /// + /// The file system path to store the generated files at. + /// Optional prefix for the generated file names. + /// The list of all saved files. + public override string[] Save(string path, string? prefix = null) => throw new NotImplementedException(); + + /// + /// Gets a trainer object to use in training the model. + /// + public override Trainer? GetTrainer() => throw new NotImplementedException(); + + /// + /// Return true if the char is valid in the tokenizer; otherwise return false. + /// + /// + /// + public override bool IsValidChar(char ch) => true; + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.Tokenizers/PreTokenizer/PreTokenizer.cs b/src/Microsoft.ML.Tokenizers/PreTokenizer/PreTokenizer.cs index a45fd26228..94acfcb96f 100644 --- a/src/Microsoft.ML.Tokenizers/PreTokenizer/PreTokenizer.cs +++ b/src/Microsoft.ML.Tokenizers/PreTokenizer/PreTokenizer.cs @@ -3,8 +3,10 @@ // See the LICENSE file in the project root for more information. using System; +using System.Collections; using System.Collections.Generic; -using System.Text; +using System.Diagnostics; +using System.Text.RegularExpressions; namespace Microsoft.ML.Tokenizers { @@ -13,7 +15,7 @@ namespace Microsoft.ML.Tokenizers /// in the original string. These offsets are in the `original` referential. /// It also contains any `Token` associated to the current split. /// - public sealed class Split : IEquatable + public readonly struct Split : IEquatable { /// /// Gets the underlying split token. Each SubString is represented by a token @@ -30,19 +32,28 @@ public sealed class Split : IEquatable /// /// create a Split object using the token and the offset /// - public Split(string token, (int Index, int End) offset) + /// The token string + /// The offset mapping to the original string + /// Indicates whether the token is a special token + public Split(string token, (int Index, int End) offset, bool isSpecialToken = false) { TokenString = token; Offset = offset; + IsSpecialToken = isSpecialToken; } + /// + /// Gets if the current Split is a special token. + /// + public bool IsSpecialToken { get; } + /// /// Indicates whether the current Split object is equal to another Split object. /// /// The Split object to compare with the current object. - public bool Equals(Split? other) => - other is not null && + public bool Equals(Split other) => TokenString == other.TokenString && + IsSpecialToken == other.IsSpecialToken && Offset.Index == other.Offset.Index && Offset.End == other.Offset.End; } @@ -54,11 +65,98 @@ other is not null && /// public abstract class PreTokenizer { + internal static readonly IReadOnlyList EmptyList = new List(); + /// /// Splits the given string in multiple substrings at the word boundary, keeping track of the offsets of said substrings from the original string. /// /// The string to split into tokens. + /// Indicates whether to skip the special tokens. /// The list of the splits containing the tokens and the token's offsets to the original string. - public abstract IReadOnlyList PreTokenize(string sentence); + public abstract IEnumerable PreTokenize(string sentence, bool skipSpecialTokens = false); + } + + internal sealed class RegexSplitEnumerable : IEnumerable + { + private readonly static Dictionary _regexCache = new(StringComparer.Ordinal); + private readonly Regex _regex; + private readonly string _sentence; + + public RegexSplitEnumerable(string sentence, string pattern) + { + Debug.Assert(sentence is not null); + Debug.Assert(pattern is not null); + + Regex? regex; + lock (_regexCache) + { + if (!_regexCache.TryGetValue(pattern!, out regex)) + { + regex = new Regex(pattern, RegexOptions.Compiled); + _regexCache[pattern!] = regex; + } + } + + _regex = regex; + _sentence = sentence!; + } + + public IEnumerator GetEnumerator() => new RegexSplitEnumerator(_regex, _sentence); + + IEnumerator IEnumerable.GetEnumerator() => new RegexSplitEnumerator(_regex, _sentence); + + private sealed class RegexSplitEnumerator : IEnumerator + { + private Split _current = default; + private readonly Regex _regex; + private Match? _tokenMatch; + private readonly string _sentence; + + public RegexSplitEnumerator(Regex regex, string sentence) + { + Debug.Assert(sentence is not null); + Debug.Assert(regex is not null); + + _regex = regex!; + _sentence = sentence!; + } + + public Split Current => _current; + + object IEnumerator.Current => _current; + + public bool MoveNext() + { + if (_tokenMatch is null) + { + _tokenMatch = _regex.Match(_sentence); + } + else if (!_tokenMatch.Success) + { + return false; + } + else + { + _tokenMatch = _tokenMatch.NextMatch(); + } + + if (!_tokenMatch.Success) + { + return false; + } + + _current = new Split(_tokenMatch.Value, (_tokenMatch.Index, _tokenMatch.Index + _tokenMatch.Length)); + return true; + } + + public void Reset() + { + _tokenMatch = null; + } + + public void Dispose() + { + } + } } } diff --git a/src/Microsoft.ML.Tokenizers/PreTokenizer/Roberta.cs b/src/Microsoft.ML.Tokenizers/PreTokenizer/Roberta.cs index 56a306df54..e07e755c29 100644 --- a/src/Microsoft.ML.Tokenizers/PreTokenizer/Roberta.cs +++ b/src/Microsoft.ML.Tokenizers/PreTokenizer/Roberta.cs @@ -4,8 +4,6 @@ using System; using System.Collections.Generic; -using System.Text; -using System.Text.RegularExpressions; namespace Microsoft.ML.Tokenizers { @@ -21,28 +19,20 @@ public sealed class RobertaPreTokenizer : PreTokenizer private const string Pattern = @"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"; - private static readonly IReadOnlyList _emptyList = new List(); - /// /// Splits the given string in multiple substrings at the word boundary, keeping track of the offsets of said substrings from the original string. /// /// The string to split into tokens. + /// Indicates whether to skip the special tokens. /// The list of the splits containing the tokens and the token's offsets to the original string. - public override IReadOnlyList PreTokenize(string? sentence) + public override IEnumerable PreTokenize(string sentence, bool skipSpecialTokens = false) { - if (sentence is null) - { - return _emptyList; - } - - List parts = new List(); - - foreach (Match match in Regex.Matches(sentence, Pattern)) + if (string.IsNullOrEmpty(sentence)) { - parts.Add(new Split(match.Value, (match.Index, match.Index + match.Length))); + return EmptyList; } - return parts; + return new RegexSplitEnumerable(sentence, Pattern); } } } diff --git a/src/Microsoft.ML.Tokenizers/PreTokenizer/TikTokenPreTokenizer.cs b/src/Microsoft.ML.Tokenizers/PreTokenizer/TikTokenPreTokenizer.cs new file mode 100644 index 0000000000..b64096de71 --- /dev/null +++ b/src/Microsoft.ML.Tokenizers/PreTokenizer/TikTokenPreTokenizer.cs @@ -0,0 +1,182 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text.RegularExpressions; + +namespace Microsoft.ML.Tokenizers +{ + /// + /// The pre-tokenizer for Tiktoken tokenizer. + /// + public sealed class TikTokenPreTokenizer : PreTokenizer + { + private readonly Regex? _specialTokensRegex; + private readonly Regex _regex; + + /// + /// Initializes a new instance of the class. + /// + /// The regex to use for splitting the text into smaller tokens in the pre-tokenization process. + /// Encode the special token to Id. + /// When regex is null + public TikTokenPreTokenizer(Regex regex, IReadOnlyDictionary? specialTokensEncoder) + { + if (regex is null) + { + throw new ArgumentNullException(nameof(regex)); + } + + _regex = regex; + + if (specialTokensEncoder is not null && specialTokensEncoder.Count > 0) + { + _specialTokensRegex = new Regex(string.Join("|", specialTokensEncoder.Keys.Select(s => Regex.Escape(s))), RegexOptions.Compiled); + } + } + + /// + /// Splits the given string in multiple substrings at the word boundary, keeping track of the offsets of said substrings from the original string. + /// + /// The string to split into tokens. + /// Indicates whether to skip the special tokens. + /// The list of the splits containing the tokens and the token's offsets to the original string. + public override IEnumerable PreTokenize(string sentence, bool skipSpecialTokens = false) + { + if (string.IsNullOrEmpty(sentence)) + { + return EmptyList; + } + + return new TokenizationEnumerable(sentence, _regex, skipSpecialTokens ? null : _specialTokensRegex); + } + + private sealed class TokenizationEnumerable : IEnumerable + { + private readonly string _sentence; + private readonly Regex _regex; + private readonly Regex? _specialTokensRegex; + + public TokenizationEnumerable(string sentence, Regex regex, Regex? specialTokensRegex) + { + if (sentence is null) + { + throw new ArgumentNullException(nameof(sentence)); + } + + if (regex is null) + { + throw new ArgumentNullException(nameof(regex)); + } + + _sentence = sentence; + _regex = regex; + _specialTokensRegex = specialTokensRegex; + } + + public IEnumerator GetEnumerator() => new TokenizationEnumerator(_sentence, _regex, _specialTokensRegex); + IEnumerator IEnumerable.GetEnumerator() => new TokenizationEnumerator(_sentence, _regex, _specialTokensRegex); + + private sealed class TokenizationEnumerator : IEnumerator + { + private Split _current = default; + private int _startIndex; + private int _offset; + private MatchCollection? _matches; + private int _matchIndex; + private Match? _specialTokenMatch; + private readonly Regex _regex; + private readonly string _sentence; + private readonly Regex? _specialTokensRegex; + + public TokenizationEnumerator(string sentence, Regex regex, Regex? specialTokensRegex) + { + Debug.Assert(sentence is not null); + Debug.Assert(regex is not null); + + _sentence = sentence!; + _regex = regex!; + _specialTokensRegex = specialTokensRegex; + _startIndex = 0; + _offset = 0; + } + + object IEnumerator.Current => _current; + + Split IEnumerator.Current => _current; + + public bool MoveNext() + { + if (_matches is not null && _matchIndex < _matches.Count) + { + Match match = _matches[_matchIndex]; + _current = new Split(match.Value, (match.Index + _offset, match.Index + _offset + match.Length), false); + _startIndex += match.Length; + _matchIndex++; + return true; + } + + if (_specialTokenMatch is not null && _specialTokenMatch.Success) + { + _current = new Split(_specialTokenMatch.Value, (_specialTokenMatch.Index, _specialTokenMatch.Index + _specialTokenMatch.Length), true); + _startIndex += _specialTokenMatch.Length; + _specialTokenMatch = null; + return true; + } + + if (_startIndex >= _sentence.Length) + { + return false; + } + + if (_specialTokensRegex is not null) + { + _specialTokenMatch = _specialTokensRegex.Match(_sentence, _startIndex); + _offset = _startIndex; + _matches = _regex.Matches(_sentence.Substring(_startIndex, _specialTokenMatch.Success ? _specialTokenMatch.Index - _startIndex : _sentence.Length - _startIndex)); + } + else + { + _matches = _regex.Matches(_sentence); + } + + if (_matches.Count > 0) + { + Match match = _matches[0]; + _current = new Split(match.Value, (match.Index + _startIndex, match.Index + _startIndex + match.Length), false); + _startIndex += match.Length; + _matchIndex = 1; + return true; + } + else if (_specialTokenMatch is not null && _specialTokenMatch.Success) + { + _current = new Split(_specialTokenMatch.Value, (_specialTokenMatch.Index, _specialTokenMatch.Index + _specialTokenMatch.Length), true); + _startIndex += _specialTokenMatch.Length; + _specialTokenMatch = null; + return true; + } + + return false; + } + + public void Reset() + { + _current = default; + _startIndex = 0; + _matches = null; + _matchIndex = -1; + _specialTokenMatch = null; + } + + public void Dispose() + { + } + } + } + } +} diff --git a/src/Microsoft.ML.Tokenizers/PreTokenizer/Whitespace.cs b/src/Microsoft.ML.Tokenizers/PreTokenizer/Whitespace.cs index 65bc2b7eea..d2d0158885 100644 --- a/src/Microsoft.ML.Tokenizers/PreTokenizer/Whitespace.cs +++ b/src/Microsoft.ML.Tokenizers/PreTokenizer/Whitespace.cs @@ -4,8 +4,6 @@ using System; using System.Collections.Generic; -using System.Text; -using System.Text.RegularExpressions; namespace Microsoft.ML.Tokenizers { @@ -22,28 +20,20 @@ public sealed class WhiteSpace : PreTokenizer private const string Pattern = @"\w+|[^\w\s]+"; - private static readonly IReadOnlyList _emptyList = new List(); - /// /// Splits the given string in multiple substrings at the word boundary, keeping track of the offsets of said substrings from the original string. /// /// The string to split into tokens. + /// Indicates whether to skip the special tokens. /// The list of the splits containing the tokens and the token's offsets to the original string. - public override IReadOnlyList PreTokenize(string? sentence) + public override IEnumerable PreTokenize(string sentence, bool skipSpecialTokens = false) { - if (sentence is null) - { - return _emptyList; - } - - List parts = new List(); - - foreach (Match match in Regex.Matches(sentence, Pattern)) + if (string.IsNullOrEmpty(sentence)) { - parts.Add(new Split(match.Value, (match.Index, match.Index + match.Length))); + return EmptyList; } - return parts; + return new RegexSplitEnumerable(sentence, Pattern); } } } diff --git a/src/Microsoft.ML.Tokenizers/Tokenizer.cs b/src/Microsoft.ML.Tokenizers/Tokenizer.cs index e4a2942364..94d07abb4d 100644 --- a/src/Microsoft.ML.Tokenizers/Tokenizer.cs +++ b/src/Microsoft.ML.Tokenizers/Tokenizer.cs @@ -6,6 +6,10 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text.RegularExpressions; +using System.Threading.Tasks; namespace Microsoft.ML.Tokenizers { @@ -52,7 +56,15 @@ public Tokenizer(Model model, PreTokenizer? preTokenizer = null, Normalizer? nor /// /// The text to tokenize. /// The tokenization result includes the tokens list, tokens Ids, tokens offset mapping. - public TokenizerResult Encode(string sequence) + public TokenizerResult Encode(string sequence) => Encode(sequence, skipSpecialTokens: false); + + /// + /// Encodes input text to object has the tokens list, tokens Ids, tokens offset mapping. + /// + /// The text to tokenize. + /// Indicate if want to skip the special tokens during the encoding. + /// The tokenization result includes the tokens list, tokens Ids, tokens offset mapping. + public TokenizerResult Encode(string sequence, bool skipSpecialTokens = false) { if (sequence is null) { @@ -75,14 +87,14 @@ public TokenizerResult Encode(string sequence) normalized = sequence; } - TokenizerResult encoding = new(sequence, normalized, PreTokenizer.PreTokenize(normalized), offsetsMappedToOriginal); + TokenizerResult encoding = new(sequence, normalized, PreTokenizer.PreTokenize(normalized, skipSpecialTokens), offsetsMappedToOriginal); if (Normalizer is null || !normalizedString.CanMapToOriginal || normalizedString.IsOneToOneMapping) { // Optimize the case we don't have to map the offsets. foreach (Split split in encoding.Splits) { - IReadOnlyList tokens = Model.Tokenize(split.TokenString); + IReadOnlyList tokens = Model.Tokenize(split.TokenString, split.IsSpecialToken); foreach (Token token in tokens) { token.Offset = (token.Offset.Index + split.Offset.Index, token.Offset.End + split.Offset.Index); @@ -97,7 +109,7 @@ public TokenizerResult Encode(string sequence) foreach (Split split in encoding.Splits) { - IReadOnlyList tokens = Model.Tokenize(split.TokenString); + IReadOnlyList tokens = Model.Tokenize(split.TokenString, split.IsSpecialToken); foreach (Token token in tokens) { int index = normalizedString.NormalizedToOriginalMapping![token.Offset.Index + split.Offset.Index]; @@ -115,6 +127,48 @@ public TokenizerResult Encode(string sequence) return encoding; } + /// + /// Encodes input text to tokens Ids. + /// + /// The text to tokenize. + /// Indicate if want to skip the special tokens during the encoding. + /// The tokenization result includes the tokens list, tokens Ids, tokens offset mapping. + public IReadOnlyList EncodeToIds(string sequence, bool skipSpecialTokens = false) + { + if (sequence is null) + { + throw new ArgumentNullException(nameof(sequence)); + } + + string normalized; + NormalizedString normalizedString = default; + + bool offsetsMappedToOriginal = true; + if (Normalizer is not null) + { + normalizedString = Normalizer.Normalize(sequence); + normalized = normalizedString.Normalized; + + offsetsMappedToOriginal = normalizedString.CanMapToOriginal; + } + else + { + normalized = sequence; + } + + List idsList = new(); + + foreach (Split split in PreTokenizer.PreTokenize(normalized, skipSpecialTokens)) + { + if (!Model.TokenizeToIds(split.TokenString, split.IsSpecialToken, idsList)) + { + throw new ArgumentException($"Unable to tokenize the sequence: {split.TokenString}"); + } + } + + return idsList; + } + // skipSpecialTokens is used in post processing we don't support yet. We are keeping it to allow using it when we support post processing. /// /// Decodes the Id to the mapped token. @@ -133,6 +187,16 @@ public TokenizerResult Encode(string sequence) /// The decoded string. public string? Decode(IEnumerable ids, bool skipSpecialTokens = false) { + if (Model is Tiktoken tiktoken) + { + // Tiktoken does not ensure a one-to-one mapping between IDs and tokens. Consequently, decoding individual IDs into tokens is not supported; + // instead, decoding all IDs must be done collectively. + // Here is example of case that map one character to multiple Ids: + // '⭐' U-2B50 is mapped to Ids [2928, 99834] in the Tiktoken model. + // In other words, the character '⭐' has UTF-8 code point 0xE2, 0xAD, 0x90, Tiktoken will map 0xE2 to [2928] and 0xAD, 0x90 to [99834]. + return tiktoken.IdsToString(ids, skipSpecialTokens); + } + List tokens = new List(); foreach (int id in ids) @@ -171,8 +235,9 @@ public void TrainFromFiles( t.Feed(lines, (s) => { string current = Normalizer is null ? s : Normalizer.Normalize(s).Normalized; - IReadOnlyList splits = PreTokenizer.PreTokenize(current); - List list = new(splits.Count); + IEnumerable splits = PreTokenizer.PreTokenize(current); + + List list = new(); foreach (Split split in splits) { list.Add(split.TokenString); @@ -195,5 +260,202 @@ public bool IsValidChar(char ch) { return Model.IsValidChar(ch); } + + private const string EndOfText = "<|endoftext|>"; + private const string FimPrefix = "<|fim_prefix|>"; + private const string FimMiddle = "<|fim_middle|>"; + private const string FimSuffix = "<|fim_suffix|>"; + private const string EndOfPrompt = "<|endofprompt|>"; + + private static readonly HttpClient _httpClient = new HttpClient(); + + private enum ModelEncoding + { + None, + Cl100kBase, + P50kBase, + P50kEdit, + R50kBase, + GPT2 + } + + private static readonly IReadOnlyDictionary _modelPrefixToEncoding = + new Dictionary() + { + // chat + { "gpt-4-", ModelEncoding.Cl100kBase }, // e.g., gpt-4-0314, etc., plus gpt-4-32k + { "gpt-3.5-turbo-", ModelEncoding.Cl100kBase } // e.g, gpt-3.5-turbo-0301, -0401, etc. + }; + + private static readonly IReadOnlyDictionary _modelToEncoding = + new Dictionary(StringComparer.OrdinalIgnoreCase) + { + // chat + { "gpt-4", ModelEncoding.Cl100kBase }, + { "gpt-3.5-turbo", ModelEncoding.Cl100kBase }, + + // text + { "text-davinci-003", ModelEncoding.P50kBase }, + { "text-davinci-002", ModelEncoding.P50kBase }, + { "text-davinci-001", ModelEncoding.R50kBase }, + { "text-curie-001", ModelEncoding.R50kBase }, + { "text-babbage-001", ModelEncoding.R50kBase }, + { "text-ada-001", ModelEncoding.R50kBase }, + { "davinci", ModelEncoding.R50kBase }, + { "curie", ModelEncoding.R50kBase }, + { "babbage", ModelEncoding.R50kBase }, + { "ada", ModelEncoding.R50kBase }, + + // code + { "code-davinci-002", ModelEncoding.P50kBase }, + { "code-davinci-001", ModelEncoding.P50kBase }, + { "code-cushman-002", ModelEncoding.P50kBase }, + { "code-cushman-001", ModelEncoding.P50kBase }, + { "davinci-codex", ModelEncoding.P50kBase }, + { "cushman-codex", ModelEncoding.P50kBase }, + + // edit + { "text-davinci-edit-001", ModelEncoding.P50kEdit }, + { "code-davinci-edit-001", ModelEncoding.P50kEdit }, + + // embeddings + { "text-embedding-ada-002", ModelEncoding.Cl100kBase }, + + // old embeddings + { "text-similarity-davinci-001", ModelEncoding.R50kBase }, + { "text-similarity-curie-001", ModelEncoding.R50kBase }, + { "text-similarity-babbage-001", ModelEncoding.R50kBase }, + { "text-similarity-ada-001", ModelEncoding.R50kBase }, + { "text-search-davinci-doc-001", ModelEncoding.R50kBase }, + { "text-search-curie-doc-001", ModelEncoding.R50kBase }, + { "text-search-babbage-doc-001", ModelEncoding.R50kBase }, + { "text-search-ada-doc-001", ModelEncoding.R50kBase }, + { "code-search-babbage-code-001", ModelEncoding.R50kBase }, + { "code-search-ada-code-001", ModelEncoding.R50kBase }, + + //open source + { "gpt2", ModelEncoding.GPT2 } + }; + + + /// + /// Create tokenizer based on model name + /// + /// Model name + /// Extra special tokens other than the built-in ones for the model + /// To normalize the text before tokenization + /// The tokenizer + public static async Task CreateByModelNameAsync( + string modelName, + IReadOnlyDictionary? extraSpecialTokens = null, + Normalizer? normalizer = null) + { + var encoder = ModelEncoding.None; + + if (!_modelToEncoding.TryGetValue(modelName, out encoder)) + { + foreach (KeyValuePair kvp in _modelPrefixToEncoding) + { + if (modelName.StartsWith(kvp.Key, StringComparison.OrdinalIgnoreCase)) + { + encoder = kvp.Value; + break; + } + } + } + + if (encoder == ModelEncoding.None) + { + throw new NotImplementedException($"Doesn't support this model [{modelName}]"); + } + + return await CreateByEncoderNameAsync(encoder, extraSpecialTokens, normalizer); + } + + private const string Cl100kBaseRegexPattern = @"(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"; + private const string P50kBaseRegexPattern = @"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"; + + const string Cl100kBaseVocabUrl = @"https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken"; + const string P50RegexUrl = @"https://openaipublic.blob.core.windows.net/encodings/p50k_base.tiktoken"; + const string R50RegexUrl = @"https://openaipublic.blob.core.windows.net/encodings/r50k_base.tiktoken"; + const string GPT2Url = @"https://pythia.blob.core.windows.net/public/encoding/gpt2.tiktoken"; + + /// + /// Create tokenizer based on encoder name and extra special tokens + /// + /// Encoder label + /// Extra special tokens other than the built-in ones for the encoder + /// To normalize the text before tokenization + /// The tokenizer + /// Throws if the encoder is not supported + private static async Task CreateByEncoderNameAsync( + ModelEncoding modelEncoding, + IReadOnlyDictionary? extraSpecialTokens, + Normalizer? normalizer) + { + switch (modelEncoding) + { + case ModelEncoding.Cl100kBase: + var specialTokens = new Dictionary + { { EndOfText, 100257}, { FimPrefix, 100258}, { FimMiddle, 100259}, { FimSuffix, 100260}, { EndOfPrompt, 100276} }; + return await CreateTikTokenTokenizerAsync(Cl100kBaseRegexPattern, Cl100kBaseVocabUrl, specialTokens, extraSpecialTokens, normalizer); + + case ModelEncoding.P50kBase: + specialTokens = new Dictionary { { EndOfText, 50256 } }; + return await CreateTikTokenTokenizerAsync(P50kBaseRegexPattern, P50RegexUrl, specialTokens, extraSpecialTokens, normalizer); + + case ModelEncoding.P50kEdit: + specialTokens = new Dictionary + { { EndOfText, 50256 }, { FimPrefix, 50281 }, { FimMiddle, 50282 }, { FimSuffix, 50283 } }; + return await CreateTikTokenTokenizerAsync(P50kBaseRegexPattern, P50RegexUrl, specialTokens, extraSpecialTokens, normalizer); + + case ModelEncoding.R50kBase: + specialTokens = new Dictionary { { EndOfText, 50256 } }; + return await CreateTikTokenTokenizerAsync(P50kBaseRegexPattern, R50RegexUrl, specialTokens, extraSpecialTokens, normalizer); + + case ModelEncoding.GPT2: + specialTokens = new Dictionary { { EndOfText, 50256 }, }; + return await CreateTikTokenTokenizerAsync(P50kBaseRegexPattern, GPT2Url, specialTokens, extraSpecialTokens, normalizer); + + default: + Debug.Assert(false, $"Unexpected encoder [{modelEncoding}]"); + throw new NotImplementedException($"Doesn't support this encoder [{modelEncoding}]"); + } + } + + private static readonly Dictionary, Dictionary, IReadOnlyDictionary)> _tiktokenCache = new(StringComparer.OrdinalIgnoreCase); + + /// + /// Create tokenizer based on regex pattern, BPE rank file and special tokens + /// + /// Regex pattern to break a long string + /// BPE rank file + /// Special tokens mapping + /// Extra special tokens other than the built-in ones for the encoder + /// To normalize the text before tokenization + /// The tokenizer + private static async Task CreateTikTokenTokenizerAsync( + string regexPatternStr, + string mergeableRanksFileUrl, + Dictionary specialTokens, + IReadOnlyDictionary? extraSpecialTokens, + Normalizer? normalizer) + { + if (extraSpecialTokens is not null) + { + specialTokens = specialTokens.Concat(extraSpecialTokens).ToDictionary(pair => pair.Key, pair => pair.Value); + } + + if (!_tiktokenCache.TryGetValue(mergeableRanksFileUrl, out (Dictionary encoder, Dictionary vocab, IReadOnlyDictionary decoder) cache)) + { + using (Stream stream = await _httpClient.GetStreamAsync(mergeableRanksFileUrl)) + { + cache = Tiktoken.LoadTikTokenBpe(stream); + } + _tiktokenCache.Add(mergeableRanksFileUrl, cache); + } + + return new Tokenizer(new Tiktoken(cache.encoder, cache.decoder, cache.vocab, specialTokens), new TikTokenPreTokenizer(new Regex(regexPatternStr, RegexOptions.Compiled), specialTokens), normalizer); + } } } diff --git a/src/Microsoft.ML.Tokenizers/TokenizerResult.cs b/src/Microsoft.ML.Tokenizers/TokenizerResult.cs index 4571e79482..192c215eec 100644 --- a/src/Microsoft.ML.Tokenizers/TokenizerResult.cs +++ b/src/Microsoft.ML.Tokenizers/TokenizerResult.cs @@ -20,7 +20,7 @@ public sealed class TokenizerResult /// The list of tokens to merge. /// The list of tokens to merge. /// Indicate whether the offsets is mapped to the original string or the normalized string. - public TokenizerResult(string originalString, string normalizedString, IReadOnlyList splits, bool offsetsMappedToOriginalString) + public TokenizerResult(string originalString, string normalizedString, IEnumerable splits, bool offsetsMappedToOriginalString) { OriginalString = originalString; NormalizedString = normalizedString; @@ -43,7 +43,7 @@ public TokenizerResult(string originalString, string normalizedString, IReadOnly /// public bool OffsetsMappedToOriginalString { get; } - internal IReadOnlyList Splits { get; } + internal IEnumerable Splits { get; } private List? _tokens; private List? _tokensWords; private List? _ids; diff --git a/src/Microsoft.ML.Tokenizers/Utils/ByteArrayComparer.cs b/src/Microsoft.ML.Tokenizers/Utils/ByteArrayComparer.cs new file mode 100644 index 0000000000..9ccca49b89 --- /dev/null +++ b/src/Microsoft.ML.Tokenizers/Utils/ByteArrayComparer.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.ML.Tokenizers +{ + internal class ByteArrayComparer : IEqualityComparer + { + public bool Equals(byte[]? x, byte[]? y) + { + if (x is null || y is null) + { + return x == y; + } + + return x.SequenceEqual(y); + } + + public int GetHashCode(byte[] bytes) + { + if (bytes == null) + { + throw new ArgumentNullException(nameof(bytes)); + } + + int hash = 17; + foreach (byte b in bytes) + { + hash = hash * 31 + b; + } + + return hash; + } + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.Tokenizers/Utils/BytePairEncoder.cs b/src/Microsoft.ML.Tokenizers/Utils/BytePairEncoder.cs new file mode 100644 index 0000000000..07a4c7db3b --- /dev/null +++ b/src/Microsoft.ML.Tokenizers/Utils/BytePairEncoder.cs @@ -0,0 +1,90 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +namespace Microsoft.ML.Tokenizers +{ + /// + /// This class implements the byte pair encoding algorithm. + /// + internal static class BytePairEncoder + { + public static int[] BytePairEncode(byte[] mergingBytes, IReadOnlyDictionary ranks) + { + if (mergingBytes.Length == 1) + { + return new int[] { ranks[mergingBytes] }; + } + + var byteIndicesAndRanks = new List<(int Index, int Rank)>(); + for (int i = 0; i < mergingBytes.Length + 1; i++) + { + byteIndicesAndRanks.Add((i, int.MaxValue)); + } + int GetRank(int startIndex, int skip = 0) + { + if (startIndex + skip + 2 < byteIndicesAndRanks.Count) + { + var slice = mergingBytes.Slice(byteIndicesAndRanks[startIndex].Index, byteIndicesAndRanks[startIndex + skip + 2].Index); + if (ranks.TryGetValue(slice, out var rank)) + { + return rank; + } + } + return int.MaxValue; + } + for (int i = 0; i < byteIndicesAndRanks.Count - 2; i++) + { + var rank = GetRank(i); + if (rank != int.MaxValue) + { + byteIndicesAndRanks[i] = (byteIndicesAndRanks[i].Index, rank); + } + } + while (byteIndicesAndRanks.Count > 1) + { + var minRank = (Index: 0, Rank: int.MaxValue); + for (int i = 0; i < byteIndicesAndRanks.Count - 1; i++) + { + if (byteIndicesAndRanks[i].Rank < minRank.Rank) + { + minRank = (i, byteIndicesAndRanks[i].Rank); + } + } + if (minRank.Rank != int.MaxValue) + { + int j = minRank.Index; + byteIndicesAndRanks[j] = (byteIndicesAndRanks[j].Index, GetRank(j, 1)); + if (j > 0) + { + byteIndicesAndRanks[j - 1] = (byteIndicesAndRanks[j - 1].Index, GetRank(j - 1, 1)); + } + byteIndicesAndRanks.RemoveAt(j + 1); + } + else + { + break; + } + } + + var outList = new int[byteIndicesAndRanks.Count - 1]; + for (int i = 0; i < byteIndicesAndRanks.Count - 1; i++) + { + outList[i] = ranks[mergingBytes.Slice(byteIndicesAndRanks[i].Index, byteIndicesAndRanks[i + 1].Index)]; + } + return outList; + } + + private static T[] Slice(this T[] array, int start, int end) + { + var length = end - start; + var result = new T[length]; + Array.Copy(array, start, result, 0, length); + return result; + } + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.Tokenizers/Utils/Helpers.netcoreapp.cs b/src/Microsoft.ML.Tokenizers/Utils/Helpers.netcoreapp.cs new file mode 100644 index 0000000000..99d764a9cf --- /dev/null +++ b/src/Microsoft.ML.Tokenizers/Utils/Helpers.netcoreapp.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Globalization; + +namespace Microsoft.ML.Tokenizers +{ + internal static class Helpers + { + public static byte[] FromBase64String(string base64String, int offset, int length) + { + Span bytes = stackalloc byte[300]; + if (!Convert.TryFromBase64Chars(base64String.AsSpan().Slice(offset, length), bytes, out int bytesWritten)) + { + throw new System.FormatException($"Invalid base64 string '{base64String.Substring(offset, length)}'"); + } + return bytes.Slice(0, bytesWritten).ToArray(); + } + + internal static bool TryParseInt32(string s, int offset, out int result) + => int.TryParse(s.AsSpan().Slice(offset), NumberStyles.None, CultureInfo.InvariantCulture, out result); + } +} + diff --git a/src/Microsoft.ML.Tokenizers/Utils/Helpers.netstandard.cs b/src/Microsoft.ML.Tokenizers/Utils/Helpers.netstandard.cs new file mode 100644 index 0000000000..4f354cda5a --- /dev/null +++ b/src/Microsoft.ML.Tokenizers/Utils/Helpers.netstandard.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; + +namespace Microsoft.ML.Tokenizers +{ + internal static class Helpers + { + public static byte[] FromBase64String(string base64String, int offset, int length) => Convert.FromBase64String(base64String.Substring(offset, length)); + + // Not support signed number + internal static bool TryParseInt32(string s, int offset, out int result) + { + result = 0; + if ((uint)offset >= s.Length) + { + return false; + } + + for (int i = offset; i < s.Length; i++) + { + if ((uint)(s[i] - '0') > ('9' - '0')) + { + return false; + } + + result = result * 10 + (s[i] - '0'); + } + + return true; + } + } +} + diff --git a/src/Microsoft.ML.Tokenizers/Utils/IListExtensions.cs b/src/Microsoft.ML.Tokenizers/Utils/IListExtensions.cs new file mode 100644 index 0000000000..061f4cc876 --- /dev/null +++ b/src/Microsoft.ML.Tokenizers/Utils/IListExtensions.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; + +namespace Microsoft.ML.Tokenizers +{ + internal static class IListExtensions + { + public static void AddRange(this IList list, IEnumerable items) + { + if (list is List concreteList) + { + concreteList.AddRange(items); + } + else + { + foreach (var item in items) + { + list.Add(item); + } + } + } + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.Tokenizers/Utils/LruCache.cs b/src/Microsoft.ML.Tokenizers/Utils/LruCache.cs new file mode 100644 index 0000000000..9ad88e2f35 --- /dev/null +++ b/src/Microsoft.ML.Tokenizers/Utils/LruCache.cs @@ -0,0 +1,134 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; + +namespace Microsoft.ML.Tokenizers +{ + internal class LruCache where TKey : notnull where TValue : notnull + { + /// + /// The default LRU cache size. + /// + public const int DefaultCacheSize = 8192; // 4096; + + private readonly object _lockObject = new object(); + + private class CacheItem + { + public readonly TKey Key; + public TValue Value; + + public CacheItem(TKey key, TValue value) + { + Key = key; + Value = value; + } + } + + private readonly Dictionary> _cache; + private readonly LinkedList _lruList; + private readonly int _cacheSize; + + /// + /// Constructs an object. + /// + /// + /// The maximum number of to mappings + /// that can be cached. This defaults to , which is set to + /// 4096. + /// + public LruCache(int cacheSize = DefaultCacheSize) + { + _cache = new Dictionary>(); + _lruList = new LinkedList(); + _cacheSize = cacheSize; + } + + /// + /// Retrieves the value associated with the specified key /> object. + /// + /// The object to be used as a key. + /// An out parameter that is set to the value of the key if key contains a mapping in the cache. + /// + /// true if the cache contains a mapping for key, false otherwise. + /// + public bool Lookup(TKey key, out TValue value) + { + lock (_lockObject) + { + if (_cache.TryGetValue(key, out LinkedListNode? cached)) + { + _lruList.Remove(cached); + _lruList.AddFirst(cached); + value = cached.Value.Value; + return true; + } + + value = default!; + return false; + } + } + + protected virtual void OnEviction(TValue evictedValue) { } + + private void EvictIfNeeded() + { + while (_cache.Count >= _cacheSize) + { + LinkedListNode? nodeToEvict = _lruList.Last; + _lruList.RemoveLast(); + _cache.Remove(nodeToEvict!.Value.Key); + OnEviction(nodeToEvict.Value.Value); + } + } + + /// + /// Adds or replaces a mapping in the cache. + /// + /// The key whose mapped is to be created or replaced. + /// The new value to be mapped to the . + public void Add(TKey key, TValue value) => Replace(key, value, out _); + + public bool Replace(TKey key, TValue value, out TValue oldValue) + { + lock (_lockObject) + { + return ReplaceInternal(key, value, out oldValue); + } + } + + private bool ReplaceInternal(TKey key, TValue value, out TValue oldValue) + { + if (_cache.TryGetValue(key, out LinkedListNode? cached)) + { + oldValue = cached.Value.Value; + cached.Value.Value = value; + _lruList.Remove(cached); + _lruList.AddFirst(cached); + return true; + } + EvictIfNeeded(); + var node = new LinkedListNode(new CacheItem(key, value)); + _cache[key] = node; + _lruList.AddFirst(node); + oldValue = default!; + return false; + } + + /// + /// The number of entries currently present in the cache. + /// + public int Count => _cache.Count; + + /// + /// Clears the contents of this cache. + /// + public void Clear() + { + _cache.Clear(); + _lruList.Clear(); + } + } +} \ No newline at end of file diff --git a/src/Microsoft.ML.Tokenizers/Utils/TokenizerExtensions.cs b/src/Microsoft.ML.Tokenizers/Utils/TokenizerExtensions.cs index 9c398cb7b1..f9d01768e5 100644 --- a/src/Microsoft.ML.Tokenizers/Utils/TokenizerExtensions.cs +++ b/src/Microsoft.ML.Tokenizers/Utils/TokenizerExtensions.cs @@ -21,7 +21,7 @@ internal static class TokenizerExtensions return minValue < int.MaxValue ? minSource : default; } - public static TValue GetOrAdd(this Dictionary dic, TKey key, TValue setValue) + public static TValue GetOrAdd(this Dictionary dic, TKey key, TValue setValue) where TKey : notnull { if (dic.TryGetValue(key, out var value)) { @@ -32,7 +32,7 @@ public static TValue GetOrAdd(this Dictionary dic, T return setValue; } - public static IReadOnlyDictionary Reverse(this IReadOnlyDictionary source) + public static IReadOnlyDictionary Reverse(this IReadOnlyDictionary source) where TValue : notnull { Dictionary dictionary = new Dictionary(); if (source != null) @@ -46,7 +46,7 @@ public static IReadOnlyDictionary Reverse(this IRead return dictionary; } - public static SortedDictionary ReverseSorted(this IReadOnlyDictionary source) + public static SortedDictionary ReverseSorted(this IReadOnlyDictionary source) where TKey : notnull where TValue : notnull { SortedDictionary dictionary = new SortedDictionary(); if (source != null) diff --git a/test/Microsoft.ML.Tokenizers.Tests/Data/lib.rs.txt b/test/Microsoft.ML.Tokenizers.Tests/Data/lib.rs.txt new file mode 100644 index 0000000000..70009d282f --- /dev/null +++ b/test/Microsoft.ML.Tokenizers.Tests/Data/lib.rs.txt @@ -0,0 +1,605 @@ +// This check is new and seems buggy (possibly with PyO3 interaction) +#![allow(clippy::borrow_deref_ref)] + +use std::collections::HashSet; +use std::thread; + +use fancy_regex::Regex; +use pyo3::exceptions; +use pyo3::prelude::*; +use pyo3::types::{PyBytes, PyList, PyTuple}; +use pyo3::PyResult; +use rustc_hash::FxHashMap as HashMap; + +fn _byte_pair_merge( + piece: &[u8], + ranks: &HashMap, usize>, + f: impl Fn(std::ops::Range) -> T, +) -> Vec { + // This is a vector of (start, rank). + // The rank is of the byte pair starting at position start. + // The rank of the last item in the vector is not a valid value. + let mut parts: Vec<(usize, usize)> = (0..piece.len() + 1).map(|i| (i, usize::MAX)).collect(); + + let get_rank = { + #[inline(always)] + |parts: &Vec<(usize, usize)>, start_idx: usize, skip: usize| { + if (start_idx + skip + 2) < parts.len() { + ranks + .get(&piece[parts[start_idx].0..parts[start_idx + skip + 2].0]) + .copied() + } else { + None + } + } + }; + + // We look up the ranks once in the beginning and iteratively update + // them during each merge, which reduces the number of rank lookups. + for i in 0..parts.len() - 2 { + match get_rank(&parts, i, 0) { + Some(rank) => { + // usize::MAX is a sentinel value and cannot be a valid rank + debug_assert!(rank != usize::MAX); + parts[i].1 = rank; + } + None => { + continue; + } + }; + } + + // If you have n parts and m merges, this does O(mn) work. + // We could do something with a heap and do O(m log n) work. + // It is important to consider that n is often small (<100), and as such + // the cache-locality benefits outweigh the algorithmic complexity downsides + // of the `parts` vector data structure above. + + // Note that we hash bytes, not token pairs. As long as we train BPE the way we + // currently do, this is equivalent. An easy way to break this would be to decouple + // merge priority from token index or to prevent specific token merges. + loop { + if parts.len() == 1 { + break; + } + + // usize::MAX is a sentinel rank value allowing us to + // take the min more quickly + let mut min_rank: (usize, usize) = (usize::MAX, 0); + for (i, &(_, rank)) in parts[..parts.len() - 1].iter().enumerate() { + if rank < min_rank.0 { + min_rank = (rank, i); + } + } + + if min_rank.0 != usize::MAX { + let i = min_rank.1; + + // NOTE: We are about to remove parts[i + 1]. We do not do it + // yet because there are cache-locality benefits to updating + // parts[i] and parts[i-1] before removing, which could thrash + // the cache. Thus, we update the rank calculation by skipping over + // parts[i + 1], by invoking `get_rank!` with `skip = 1`. + parts[i].1 = get_rank(&parts, i, 1).unwrap_or(usize::MAX); + if i > 0 { + parts[i - 1].1 = get_rank(&parts, i - 1, 1).unwrap_or(usize::MAX); + } + + parts.remove(i + 1); + } else { + break; + } + } + let mut out: Vec = Vec::with_capacity(parts.len() - 1); + for i in 0..parts.len() - 1 { + out.push(f(parts[i].0..parts[i + 1].0)); + } + out +} + +pub fn byte_pair_encode(piece: &[u8], ranks: &HashMap, usize>) -> Vec { + if piece.len() == 1 { + return vec![ranks[piece]]; + } + _byte_pair_merge(piece, ranks, |p| ranks[&piece[p.start..p.end]]) +} + +pub fn byte_pair_split<'a>(piece: &'a [u8], ranks: &HashMap, usize>) -> Vec<&'a [u8]> { + if piece.len() == 1 { + return vec![piece]; + } + _byte_pair_merge(piece, ranks, |p| &piece[p.start..p.end]) +} + +// Various performance notes: +// +// Regex +// ===== +// Most of the time is spent in regex. The easiest way to speed this up is by using less fancy +// regex features. For instance, using a regex parse-able by `regex` crate is 3x faster than +// the usual regex we use. +// +// However, given that we're using a regex parse-able by `regex`, there isn't much difference +// between using the `regex` crate and using the `fancy_regex` crate. +// +// There is an important interaction between threading, `regex` and `fancy_regex`. +// When using `fancy_regex`, we hit `regex.find_at`. It turns out that this causes contention on +// some mutable scratch space inside of `regex`. This absolutely kills performance. When using plain +// old `regex`, we don't hit this, because `find_iter` has a different code path. +// Related: https://github.com/rust-lang/regex/blob/master/PERFORMANCE.md +// Anyway, the way we get around this is with having a (mostly) thread local clone of the regex for +// each thread. +// +// Threading +// ========= +// I tried using `rayon`. It wasn't really faster than using Python threads and releasing the GIL. +// So goodbye `rayon`! Let thread count etc be in control of our Python users. +// +// Caching +// ======= +// The reference tokeniser has an lru cache over the equivalent of `byte_pair_encode`. +// Originally, we had one too! Without it, we were only vaguely faster than Python. +// I used an RWLock to protect the cache. This didn't seem to hurt single threaded performance +// noticeably, but it did affect multi-threaded performance. Weirdly, it seemed to affect +// multi-threaded performance even when I only had readers (maybed I messed something up?). +// Anyway, I realised that we could get rid of the cache, if we treat the set of tokens as a cache! +// These are exactly the set or merges that are likely to be hot. And now we don't have to think +// about interior mutability, memory use, or cloning. +// +// Hashing +// ======= +// We use FxHashMap instead of the standard HashMap. This is maybe like a 5-10% win? +// The current implementation ends up doing a lot of hashing of bytes. In theory, this could be made +// to be hashing of two-tuples of ints, which looks like it may also be a couple percent faster. + +use std::num::NonZeroU64; +pub struct FakeThreadId(NonZeroU64); + +fn hash_current_thread() -> usize { + // It's easier to use unsafe than to use nightly. Rust has this nice u64 thread id counter + // that works great for our use case of avoiding collisions in our array. Unfortunately, + // it's private. However, there are only so many ways you can layout a u64, so just transmute + // https://github.com/rust-lang/rust/issues/67939 + const _: [u8; 8] = [0; std::mem::size_of::()]; + const _: [u8; 8] = [0; std::mem::size_of::()]; + let x = unsafe { + std::mem::transmute::(thread::current().id()).0 + }; + u64::from(x) as usize +} + +const MAX_NUM_THREADS: usize = 128; +#[pyclass] +struct CoreBPE { + encoder: HashMap, usize>, + special_tokens_encoder: HashMap, + decoder: HashMap>, + special_tokens_decoder: HashMap>, + regex_tls: Vec, + special_regex_tls: Vec, + sorted_token_bytes: Vec>, +} + +impl CoreBPE { + fn _get_tl_regex(&self) -> &Regex { + // See performance notes above for what this is about + // It's also a little janky, please make a better version of it! + // However, it's nice that this doesn't leak memory to short-lived threads + &self.regex_tls[hash_current_thread() % MAX_NUM_THREADS] + } + + fn _get_tl_special_regex(&self) -> &Regex { + &self.special_regex_tls[hash_current_thread() % MAX_NUM_THREADS] + } + + fn _decode_native(&self, tokens: &[usize]) -> Vec { + let mut ret = Vec::with_capacity(tokens.len() * 2); + for token in tokens { + let token_bytes = self + .decoder + .get(token) + .unwrap_or_else(|| &self.special_tokens_decoder[token]); + ret.extend(token_bytes); + } + ret + } + + fn _encode_ordinary_native(&self, text: &str) -> Vec { + // This is the core of the encoding logic; the other functions in here + // just make things complicated :-) + let regex = self._get_tl_regex(); + let mut ret = vec![]; + for mat in regex.find_iter(text) { + let piece = mat.unwrap().as_str().as_bytes(); + if let Some(token) = self.encoder.get(piece) { + ret.push(*token); + continue; + } + ret.extend(&byte_pair_encode(piece, &self.encoder)); + } + ret + } + + fn _encode_native(&self, text: &str, allowed_special: &HashSet<&str>) -> (Vec, usize) { + let special_regex = self._get_tl_special_regex(); + let regex = self._get_tl_regex(); + let mut ret = vec![]; + + let mut start = 0; + let mut last_piece_token_len = 0; + loop { + let mut next_special; + let mut start_find = start; + loop { + // Find the next allowed special token, if any + next_special = special_regex.find_from_pos(text, start_find).unwrap(); + match next_special { + Some(m) => { + if allowed_special.contains(&text[m.start()..m.end()]) { + break; + } + start_find = m.start() + 1; + } + None => break, + } + } + let end = next_special.map_or(text.len(), |m| m.start()); + + // Okay, here we go, compare this logic to _encode_ordinary_native + for mat in regex.find_iter(&text[start..end]) { + let piece = mat.unwrap().as_str().as_bytes(); + if let Some(token) = self.encoder.get(piece) { + last_piece_token_len = 1; + ret.push(*token); + continue; + } + let tokens = byte_pair_encode(piece, &self.encoder); + last_piece_token_len = tokens.len(); + ret.extend(&tokens); + } + + match next_special { + // And here we push the special token + Some(m) => { + let piece = m.as_str(); + let token = self.special_tokens_encoder[piece]; + ret.push(token); + start = m.end(); + last_piece_token_len = 0; + } + None => break, + } + } + + // last_piece_token_len is how many tokens came from the last regex split. This is used + // for determining unstable tokens, since you can't merge across (stable) regex splits + (ret, last_piece_token_len) + } + + fn _increase_last_piece_token_len( + &self, + tokens: Vec, + mut last_piece_token_len: usize, + ) -> (Vec, usize) { + // Unfortunately, the locations where our regex splits can be unstable. + // For the purposes of determining unstable tokens, unstable regex splitting + // is only a problem if a split that was present disappears, since this can + // lead to merging of tokens otherwise thought to be stable. + // cl100k_base makes our life hard by including the \s*[\r\n]+ + // pattern. This can e.g. cause "\n" + " " to become "\n \n". + // Here is a quick and dirty fix: + { + let token_is_all_space = |token| { + self.decoder + .get(token) + .map(|token_bytes| { + token_bytes + .iter() + .rev() + .all(|&b| [b' ', b'\n', b'\t'].contains(&b)) + }) + .unwrap_or(false) + }; + if last_piece_token_len > 0 + && token_is_all_space(&tokens[tokens.len() - last_piece_token_len]) + { + while (last_piece_token_len < tokens.len()) + && token_is_all_space(&tokens[tokens.len() - last_piece_token_len - 1]) + { + last_piece_token_len += 1; + } + } + } + debug_assert!(last_piece_token_len <= tokens.len()); + + (tokens, last_piece_token_len) + } + + fn _encode_unstable_native( + &self, + text: &str, + allowed_special: &HashSet<&str>, + ) -> (Vec, HashSet>) { + let (tokens, last_piece_token_len) = self._encode_native(text, allowed_special); + if last_piece_token_len == 0 { + // If last_piece_token_len is zero, the last token was a special token and we have + // no unstable bytes + return (tokens, HashSet::new()); + } + let (mut tokens, last_piece_token_len) = + self._increase_last_piece_token_len(tokens, last_piece_token_len); + + let unstable_bytes = self._decode_native(&tokens[tokens.len() - last_piece_token_len..]); + tokens.truncate(tokens.len() - last_piece_token_len); + + // TODO: we should try harder to find additional stable tokens + // This would reduce the amount of retokenising when determining completions + // Refer to the logic in an older version of this file + + let mut completions = HashSet::new(); + if unstable_bytes.is_empty() { + return (tokens, completions); + } + + // This is the easy bit. Just find all single tokens that start with unstable_bytes + // (including tokens that exactly match unstable_bytes) + // Separating this from the loop below helps with performance in a common case. + let mut point = self + .sorted_token_bytes + .partition_point(|x| x.as_slice() < unstable_bytes.as_slice()); + while point < self.sorted_token_bytes.len() + && self.sorted_token_bytes[point].starts_with(&unstable_bytes) + { + completions.insert(vec![ + self.encoder[self.sorted_token_bytes[point].as_slice()], + ]); + point += 1; + } + + // Now apply even more brute force. At every (other) possible position for the straddling + // token, concatenate additional bytes from that token (if any) to unstable_bytes, + // and retokenise the whole thing and see what we get. + for i in 1..unstable_bytes.len() { + let prefix = &unstable_bytes[..i]; + let suffix = &unstable_bytes[i..]; + let mut point = self + .sorted_token_bytes + .partition_point(|x| x.as_slice() < suffix); + // TODO: Perf optimisation if suffix starts with " "? + while point < self.sorted_token_bytes.len() + && self.sorted_token_bytes[point].starts_with(suffix) + { + let possibility = [prefix, self.sorted_token_bytes[point].as_slice()].concat(); + let encoded = match std::str::from_utf8(&possibility) { + // Morally, this is byte_pair_encode(&possibility, &self.encoder) + // But we might have introduced a regex split which would prevent merges. + // (particularly possible in the presence of unstable regex splits) + // So convert to UTF-8 and do regex splitting. + // E.g. with cl100k_base " !" gets split to " " + " !", + // but byte_pair_encode(" !") != byte_pair_encode(" ") + Ok(s) => self._encode_ordinary_native(s), + + // Technically, whether or not this arm is correct depends on whether there + // would be a regex split before the UTF-8 truncation point. + // Probably niche enough that no one will ever notice (after all, people didn't + // notice all the big holes in the previous unstable token implementation) + Err(_) => byte_pair_encode(&possibility, &self.encoder), + // Something like the following is intriguing but incorrect: + // Err(e) => self._encode_ordinary_native(unsafe { + // std::str::from_utf8_unchecked(&possibility[..e.valid_up_to()]) + // }), + }; + let mut seq = Vec::new(); + let mut seq_len = 0; + for token in encoded { + seq.push(token); + seq_len += self.decoder[&token].len(); + if seq_len >= unstable_bytes.len() { + break; + } + } + completions.insert(seq); + point += 1; + } + } + + // This is also not straightforward. While we generally assume that regex splits are stable, + // unfortunately, they are not. That is, if adding bytes were to make a split appear in + // unstable_bytes, this could make tokens possible which our logic would otherwise think + // would be merged. + // For example, with gpt2, the use of \s+(?!\S) means that "\n\n" could + // develop a split, e.g. "\n\n0" splits into "\n"+"\n"+"0", making "\n" a possible token. + // Here is a quick and dirty fix: + // This isn't right if we ever remove \s+(?!\S) + if unstable_bytes.len() > 1 { + let last_decoded = bstr::decode_last_utf8(unstable_bytes.as_slice()); + if unstable_bytes.len() - last_decoded.1 > 0 + && last_decoded.0.map_or(false, |c| c.is_whitespace()) + { + let mut reencoded = byte_pair_encode( + &unstable_bytes[..unstable_bytes.len() - last_decoded.1], + &self.encoder, + ); + reencoded.extend(byte_pair_encode( + &unstable_bytes[unstable_bytes.len() - last_decoded.1..], + &self.encoder, + )); + completions.insert(reencoded); + } + } + + (tokens, completions) + } +} + +#[pymethods] +impl CoreBPE { + #[new] + fn new( + encoder: HashMap, usize>, + special_tokens_encoder: HashMap, + pattern: &str, + ) -> PyResult { + let regex = Regex::new(pattern) + .map_err(|e| PyErr::new::(e.to_string()))?; + + let special_regex = { + let _parts = special_tokens_encoder + .keys() + .map(|s| fancy_regex::escape(s)) + .collect::>(); + Regex::new(&_parts.join("|")) + .map_err(|e| PyErr::new::(e.to_string()))? + }; + + let decoder: HashMap> = + encoder.iter().map(|(k, v)| (*v, k.clone())).collect(); + + assert!(encoder.len() == decoder.len()); + + let special_tokens_decoder: HashMap> = special_tokens_encoder + .iter() + .map(|(k, v)| (*v, k.as_bytes().to_vec())) + .collect(); + + // Clone because I don't know how to tell Rust I'm not going to change the map + let mut sorted_token_bytes: Vec> = encoder.keys().cloned().collect(); + sorted_token_bytes.sort(); + + Ok(CoreBPE { + encoder, + special_tokens_encoder, + decoder, + special_tokens_decoder, + regex_tls: (0..MAX_NUM_THREADS).map(|_| regex.clone()).collect(), + special_regex_tls: (0..MAX_NUM_THREADS) + .map(|_| special_regex.clone()) + .collect(), + sorted_token_bytes, + }) + } + + // ==================== + // Encoding + // ==================== + + fn encode_ordinary(&self, py: Python, text: &str) -> Vec { + py.allow_threads(|| self._encode_ordinary_native(text)) + } + + fn encode(&self, py: Python, text: &str, allowed_special: HashSet<&str>) -> Vec { + py.allow_threads(|| self._encode_native(text, &allowed_special).0) + } + + fn _encode_bytes(&self, py: Python, bytes: &[u8]) -> Vec { + py.allow_threads(|| { + match std::str::from_utf8(bytes) { + Ok(text) => self._encode_ordinary_native(text), + Err(e) => { + let text = unsafe { std::str::from_utf8_unchecked(&bytes[..e.valid_up_to()]) }; + let (tokens, last_piece_token_len) = self._encode_native(text, &HashSet::new()); + let (mut tokens, last_piece_token_len) = + self._increase_last_piece_token_len(tokens, last_piece_token_len); + if !tokens.is_empty() && last_piece_token_len > 0 { + // Lop off the tokens from the last piece and run BPE on the remaining bytes + // Somewhat niche, but this may not be correct if we'd have had a regex + // split between the valid UTF-8 and the invalid bytes, which is why this + // method is private + let mut unstable_bytes = + self._decode_native(&tokens[tokens.len() - last_piece_token_len..]); + unstable_bytes.extend_from_slice(&bytes[e.valid_up_to()..]); + + tokens.truncate(tokens.len() - last_piece_token_len); + tokens.extend(byte_pair_encode(&unstable_bytes, &self.encoder)); + } + tokens + } + } + }) + } + + fn encode_with_unstable( + &self, + py: Python, + text: &str, + allowed_special: HashSet<&str>, + ) -> Py { + let (tokens, completions) = + py.allow_threads(|| self._encode_unstable_native(text, &allowed_special)); + let py_completions = + PyList::new(py, completions.iter().map(|seq| PyList::new(py, &seq[..]))); + (tokens, py_completions).into_py(py) + } + + fn encode_single_token(&self, piece: &[u8]) -> PyResult { + if let Some(token) = self.encoder.get(piece).copied() { + return Ok(token); + } + if let Ok(piece_str) = std::str::from_utf8(piece) { + if let Some(token) = self.special_tokens_encoder.get(piece_str).copied() { + return Ok(token); + } + } + Err(PyErr::new::(piece.to_owned())) + } + + fn encode_single_piece(&self, piece: &[u8]) -> Vec { + if let Some(token) = self.encoder.get(piece) { + return vec![*token]; + } + byte_pair_encode(piece, &self.encoder) + } + + // ==================== + // Decoding + // ==================== + + fn decode_bytes(&self, py: Python, tokens: Vec) -> Py { + let bytes = py.allow_threads(|| self._decode_native(&tokens)); + PyBytes::new(py, &bytes).into() + } + + fn decode_single_token_bytes(&self, py: Python, token: usize) -> PyResult> { + if let Some(bytes) = self.decoder.get(&token) { + return Ok(PyBytes::new(py, bytes).into()); + } + if let Some(bytes) = self.special_tokens_decoder.get(&token) { + return Ok(PyBytes::new(py, bytes).into()); + } + Err(PyErr::new::(token.to_string())) + } + + // ==================== + // Miscellaneous + // ==================== + + fn token_byte_values(&self, py: Python) -> Vec> { + self.sorted_token_bytes + .iter() + .map(|x| PyBytes::new(py, x).into()) + .collect() + } +} + +#[pymodule] +fn _tiktoken(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_class::()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use rustc_hash::FxHashMap as HashMap; + + use crate::byte_pair_split; + + #[test] + fn very_simple_test() { + let mut ranks = HashMap::default(); + ranks.insert(b"ab".to_vec(), 1); + ranks.insert(b"cd".to_vec(), 2); + + let res = byte_pair_split(b"abcd", &ranks); + assert_eq!(res, vec![b"ab", b"cd"]); + } +} diff --git a/test/Microsoft.ML.Tokenizers.Tests/Data/tokens.json b/test/Microsoft.ML.Tokenizers.Tests/Data/tokens.json new file mode 100644 index 0000000000..4d5c63f8b4 --- /dev/null +++ b/test/Microsoft.ML.Tokenizers.Tests/Data/tokens.json @@ -0,0 +1 @@ +[322, 1115, 1817, 374, 502, 323, 5084, 80411, 320, 54228, 449, 5468, 46, 18, 16628, 340, 86466, 7331, 10050, 46849, 487, 72241, 821, 45070, 7949, 28871, 817, 1487, 487, 52237, 487, 45701, 280, 817, 1487, 487, 4629, 401, 817, 27555, 42932, 487, 33564, 280, 817, 4611, 78, 18, 487, 62852, 280, 817, 4611, 78, 18, 487, 1762, 53638, 57262, 817, 4611, 78, 18, 487, 9426, 23821, 14149, 7223, 11, 5468, 861, 11, 81924, 2499, 817, 4611, 78, 18, 487, 14149, 2122, 280, 817, 23941, 66, 9127, 487, 82956, 19004, 439, 10751, 401, 8998, 721, 3867, 14784, 21558, 3203, 17492, 262, 6710, 25, 45690, 84, 23, 1282, 262, 21467, 25, 612, 19004, 51539, 35937, 23, 8226, 23098, 12803, 262, 282, 25, 11866, 51282, 5305, 487, 3806, 487, 6174, 91344, 9414, 1492, 350, 345, 8, 1492, 11564, 3203, 29, 341, 262, 443, 1115, 374, 264, 4724, 315, 320, 2527, 11, 7222, 4390, 262, 443, 578, 7222, 374, 315, 279, 5027, 6857, 6041, 520, 2361, 1212, 627, 262, 443, 578, 7222, 315, 279, 1566, 1537, 304, 279, 4724, 374, 539, 264, 2764, 907, 627, 262, 1095, 5318, 5596, 25, 11564, 29806, 52978, 11, 23098, 16401, 284, 320, 15, 497, 23164, 19970, 368, 489, 220, 16, 570, 2235, 23236, 72, 91, 320, 72, 11, 23098, 487, 10809, 4682, 17840, 1454, 262, 1095, 636, 21048, 284, 341, 286, 11765, 5167, 39677, 5680, 286, 765, 18753, 25, 612, 10256, 29806, 52978, 11, 23098, 8, 8226, 1212, 7406, 25, 23098, 11, 10936, 25, 23098, 91, 341, 310, 422, 320, 2527, 7406, 489, 10936, 489, 220, 17, 8, 366, 5596, 19970, 368, 341, 394, 21467, 198, 504, 662, 456, 2146, 23164, 58, 18753, 29563, 7406, 948, 15, 497, 18753, 29563, 7406, 489, 10936, 489, 220, 17, 948, 15, 2608, 504, 662, 38828, 1142, 746, 310, 335, 775, 341, 394, 2290, 198, 310, 457, 286, 457, 262, 3718, 262, 443, 1226, 1427, 709, 279, 21467, 3131, 304, 279, 7314, 323, 5480, 8046, 2713, 198, 262, 443, 1124, 2391, 1855, 11117, 11, 902, 26338, 279, 1396, 315, 7222, 1427, 8772, 627, 262, 369, 602, 304, 220, 15, 497, 18753, 19970, 368, 482, 220, 17, 341, 286, 2489, 636, 21048, 2146, 18753, 11, 602, 11, 220, 15, 8, 341, 310, 4427, 71976, 8, 591, 341, 394, 443, 23098, 487, 10809, 374, 264, 81878, 907, 323, 4250, 387, 264, 2764, 7222, 198, 394, 7542, 16962, 10509, 13430, 976, 23098, 487, 10809, 317, 394, 5596, 1004, 948, 16, 284, 7222, 280, 310, 457, 310, 2290, 591, 341, 394, 3136, 280, 310, 457, 286, 2670, 262, 557, 262, 443, 1442, 499, 617, 308, 5596, 323, 296, 82053, 11, 420, 1587, 507, 1278, 77, 8, 990, 627, 262, 443, 1226, 1436, 656, 2555, 449, 264, 17817, 323, 656, 507, 1278, 1515, 308, 8, 990, 627, 262, 443, 1102, 374, 3062, 311, 2980, 430, 308, 374, 3629, 2678, 23246, 1041, 705, 323, 439, 1778, 198, 262, 443, 279, 6636, 41160, 488, 7720, 63074, 279, 12384, 292, 23965, 40291, 3422, 198, 262, 443, 315, 279, 1595, 18753, 63, 4724, 828, 6070, 3485, 382, 262, 443, 7181, 430, 584, 5286, 5943, 11, 539, 4037, 13840, 13, 1666, 1317, 439, 584, 5542, 426, 1777, 279, 1648, 584, 198, 262, 443, 5131, 656, 11, 420, 374, 13890, 13, 1556, 4228, 1648, 311, 1464, 420, 1053, 387, 311, 1654, 85489, 198, 262, 443, 11117, 10844, 505, 4037, 1963, 477, 311, 5471, 3230, 4037, 82053, 627, 262, 6471, 341, 286, 422, 5596, 19970, 368, 624, 220, 16, 341, 310, 1464, 280, 286, 557, 286, 443, 23098, 487, 10809, 374, 264, 81878, 7222, 907, 10923, 603, 311, 198, 286, 443, 1935, 279, 1332, 810, 6288, 198, 286, 1095, 5318, 1332, 21048, 25, 320, 52978, 11, 23098, 8, 284, 320, 52978, 487, 10809, 11, 220, 15, 317, 286, 369, 320, 72, 11, 612, 42217, 7222, 595, 304, 5596, 96974, 18753, 19970, 368, 482, 220, 16, 948, 2058, 1020, 77669, 368, 341, 310, 422, 7222, 366, 1332, 21048, 13, 15, 341, 394, 1332, 21048, 284, 320, 13430, 11, 602, 317, 310, 457, 286, 557, 286, 422, 1332, 21048, 13, 15, 976, 23098, 487, 10809, 341, 310, 1095, 602, 284, 1332, 21048, 13, 16, 401, 310, 443, 17160, 25, 1226, 527, 922, 311, 4148, 5596, 1004, 489, 220, 16, 948, 1226, 656, 539, 656, 433, 198, 310, 443, 3686, 1606, 1070, 527, 6636, 41160, 488, 7720, 311, 21686, 198, 310, 443, 5596, 1004, 60, 323, 5596, 1004, 12, 16, 60, 1603, 18054, 11, 902, 1436, 8983, 1003, 198, 310, 443, 279, 6636, 13, 14636, 11, 584, 2713, 279, 7222, 22702, 555, 43759, 927, 198, 310, 443, 5596, 1004, 489, 220, 16, 1145, 555, 47128, 1595, 456, 21048, 0, 63, 449, 1595, 21247, 284, 220, 16, 19154, 310, 5596, 1004, 948, 16, 284, 636, 21048, 2146, 18753, 11, 602, 11, 220, 16, 570, 15818, 8908, 7, 52978, 487, 10809, 317, 310, 422, 602, 871, 220, 15, 341, 394, 5596, 1004, 482, 220, 16, 948, 16, 284, 636, 21048, 2146, 18753, 11, 602, 482, 220, 16, 11, 220, 16, 570, 15818, 8908, 7, 52978, 487, 10809, 317, 310, 557, 310, 5596, 4955, 1998, 489, 220, 16, 317, 286, 335, 775, 341, 310, 1464, 280, 286, 457, 262, 457, 262, 1095, 5318, 704, 25, 11564, 3203, 29, 284, 11564, 487, 4291, 36703, 47315, 19970, 368, 482, 220, 16, 317, 262, 369, 602, 304, 220, 15, 497, 18753, 19970, 368, 482, 220, 16, 341, 286, 704, 2615, 968, 47315, 1004, 948, 15, 497, 18753, 1004, 489, 220, 16, 948, 15, 1125, 262, 457, 262, 704, 198, 633, 9780, 5279, 5027, 14784, 11473, 83103, 25, 45690, 84, 23, 1145, 21467, 25, 612, 19004, 51539, 35937, 23, 8226, 23098, 9414, 1492, 11564, 91344, 29, 341, 262, 422, 6710, 19970, 368, 624, 220, 16, 341, 286, 471, 7638, 21362, 81, 4129, 11661, 9286, 13507, 262, 457, 262, 721, 3867, 14784, 21558, 83103, 11, 21467, 11, 765, 79, 91, 21467, 58, 5, 23164, 11661, 5069, 497, 79, 5183, 27829, 633, 9780, 5279, 5027, 14784, 17489, 18795, 64, 2284, 23164, 25, 31236, 64, 510, 84, 23, 1145, 21467, 25, 612, 19004, 51539, 35937, 23, 8226, 23098, 9414, 1492, 11564, 53344, 6, 64, 510, 84, 23, 26753, 341, 262, 422, 6710, 19970, 368, 624, 220, 16, 341, 286, 471, 7638, 21362, 23164, 947, 262, 457, 262, 721, 3867, 14784, 21558, 83103, 11, 21467, 11, 765, 79, 91, 612, 23164, 11661, 5069, 497, 79, 5183, 2608, 633, 322, 40741, 5178, 8554, 512, 2341, 322, 27238, 198, 322, 31842, 198, 322, 7648, 315, 279, 892, 374, 7543, 304, 20791, 13, 578, 30689, 1648, 311, 4732, 420, 709, 374, 555, 1701, 2753, 27555, 198, 322, 20791, 4519, 13, 1789, 2937, 11, 1701, 264, 20791, 4820, 12, 481, 555, 1595, 27485, 63, 18187, 374, 220, 18, 87, 10819, 1109, 198, 322, 279, 13783, 20791, 584, 1005, 627, 2341, 322, 4452, 11, 2728, 430, 584, 2351, 1701, 264, 20791, 4820, 12, 481, 555, 1595, 27485, 7964, 1070, 4536, 956, 1790, 6811, 198, 322, 1990, 1701, 279, 1595, 27485, 63, 18187, 323, 1701, 279, 1595, 69, 6709, 42932, 63, 18187, 627, 2341, 322, 2684, 374, 459, 3062, 16628, 1990, 31259, 11, 1595, 27485, 63, 323, 1595, 69, 6709, 42932, 19154, 322, 3277, 1701, 1595, 69, 6709, 42932, 7964, 584, 4295, 1595, 27485, 2725, 3837, 29687, 1102, 10800, 704, 430, 420, 11384, 52240, 389, 198, 322, 1063, 26889, 19307, 3634, 4871, 315, 1595, 27485, 29687, 1115, 11112, 29910, 5178, 13, 3277, 1701, 14733, 198, 322, 2362, 1595, 27485, 7964, 584, 1541, 956, 4295, 420, 11, 1606, 1595, 3990, 11985, 63, 706, 264, 2204, 2082, 1853, 627, 322, 25368, 25, 3788, 1129, 5316, 916, 7534, 592, 76560, 14, 27485, 35927, 24184, 14, 9851, 63653, 22030, 198, 322, 42669, 11, 279, 1648, 584, 636, 2212, 420, 374, 449, 3515, 264, 320, 63995, 8, 4617, 2254, 15057, 315, 279, 20791, 369, 198, 322, 1855, 4617, 627, 2341, 322, 666, 6285, 198, 322, 284, 77450, 322, 358, 6818, 1701, 1595, 11872, 263, 29687, 1102, 5828, 956, 2216, 10819, 1109, 1701, 13325, 14906, 323, 28965, 279, 480, 1750, 627, 322, 2100, 47555, 1595, 11872, 263, 63, 0, 6914, 4617, 1797, 5099, 387, 304, 2585, 315, 1057, 13325, 3932, 627, 2341, 322, 356, 12092, 198, 322, 31842, 21410, 322, 578, 5905, 4037, 12329, 706, 459, 326, 2739, 6636, 927, 279, 13890, 315, 1595, 3867, 14784, 11473, 19154, 322, 25842, 11, 584, 1047, 832, 2288, 0, 17586, 433, 11, 584, 1051, 1193, 73059, 10819, 1109, 13325, 627, 322, 358, 1511, 459, 47306, 12258, 311, 6144, 279, 6636, 13, 1115, 3287, 956, 2873, 311, 13194, 3254, 67887, 5178, 198, 322, 78988, 11, 719, 433, 1550, 7958, 7447, 61904, 291, 5178, 13, 77253, 398, 11, 433, 9508, 311, 7958, 198, 322, 7447, 61904, 291, 5178, 1524, 994, 358, 1193, 1047, 13016, 320, 18864, 2788, 358, 65302, 2555, 709, 30, 4390, 322, 42669, 11, 358, 41193, 430, 584, 1436, 636, 9463, 315, 279, 6636, 11, 422, 584, 4322, 279, 743, 315, 11460, 439, 264, 6636, 4999, 322, 4314, 527, 7041, 279, 743, 477, 82053, 430, 527, 4461, 311, 387, 4106, 13, 1628, 1457, 584, 1541, 956, 617, 311, 1781, 198, 322, 922, 15135, 5318, 2968, 11, 5044, 1005, 11, 477, 78966, 627, 2341, 322, 6668, 287, 198, 322, 31842, 21410, 322, 1226, 1005, 435, 87, 19004, 4619, 315, 279, 5410, 10751, 13, 1115, 374, 7344, 1093, 264, 220, 20, 12, 605, 4, 3243, 5380, 322, 578, 1510, 8292, 10548, 709, 3815, 264, 2763, 315, 73455, 315, 5943, 13, 763, 10334, 11, 420, 1436, 387, 1903, 198, 322, 311, 387, 73455, 315, 1403, 2442, 29423, 315, 55824, 11, 902, 5992, 1093, 433, 1253, 1101, 387, 264, 5743, 3346, 10819, 382, 817, 1487, 487, 2470, 487, 8284, 18483, 52, 1227, 280, 9780, 2080, 38065, 92170, 7, 8284, 18483, 52, 1227, 629, 8998, 5286, 11327, 11048, 368, 1492, 23098, 341, 262, 443, 1102, 596, 8831, 311, 1005, 20451, 1109, 311, 1005, 75860, 13, 34889, 706, 420, 6555, 577, 1227, 4617, 887, 5663, 198, 262, 443, 430, 4375, 2294, 369, 1057, 1005, 1162, 315, 31526, 48453, 304, 1057, 1358, 13, 19173, 345, 262, 443, 433, 596, 879, 13, 4452, 11, 1070, 527, 1193, 779, 1690, 5627, 499, 649, 6932, 264, 577, 1227, 11, 779, 1120, 1380, 53314, 198, 262, 443, 3788, 1129, 5316, 916, 7534, 592, 76560, 7534, 592, 39845, 14, 25136, 2137, 198, 262, 738, 59636, 510, 84, 23, 26, 220, 23, 60, 284, 510, 15, 26, 1487, 487, 10759, 487, 2190, 3659, 28738, 1872, 487, 4629, 487, 92170, 13867, 947, 262, 738, 59636, 510, 84, 23, 26, 220, 23, 60, 284, 510, 15, 26, 1487, 487, 10759, 487, 2190, 3659, 28738, 53417, 92170, 13867, 947, 262, 1095, 865, 284, 20451, 341, 286, 1487, 487, 10759, 487, 1485, 53314, 28738, 1872, 487, 4629, 487, 92170, 11, 38065, 92170, 2284, 4629, 487, 3311, 1020, 307, 6139, 15, 198, 262, 2670, 262, 577, 1227, 487, 1527, 2120, 8, 439, 23098, 198, 633, 1040, 8498, 9828, 57339, 25, 23098, 284, 220, 4386, 280, 13657, 3368, 1058, 933, 1257, 9708, 33, 1777, 341, 262, 24592, 25, 10751, 51539, 35937, 23, 8226, 23098, 12803, 262, 3361, 29938, 40168, 25, 10751, 3548, 11, 23098, 12803, 262, 25569, 25, 10751, 91344, 11, 11564, 35937, 23, 62440, 262, 3361, 29938, 50943, 25, 10751, 91344, 11, 11564, 35937, 23, 62440, 262, 20791, 72362, 25, 11564, 27, 33564, 12803, 262, 3361, 42932, 72362, 25, 11564, 27, 33564, 12803, 262, 10839, 6594, 12807, 25, 11564, 51539, 35937, 23, 62440, 633, 6517, 9708, 33, 1777, 341, 262, 5279, 721, 456, 84070, 42932, 2146, 726, 8, 1492, 612, 33564, 341, 286, 443, 3580, 5178, 8554, 3485, 369, 1148, 420, 374, 922, 198, 286, 443, 1102, 596, 1101, 264, 2697, 503, 92916, 11, 4587, 1304, 264, 2731, 2373, 315, 433, 4999, 286, 443, 4452, 11, 433, 596, 6555, 430, 420, 3250, 956, 24237, 5044, 311, 2875, 62954, 14906, 198, 286, 612, 726, 44218, 72362, 80941, 11327, 11048, 368, 1034, 8498, 9828, 57339, 933, 262, 557, 262, 5279, 721, 456, 84070, 42729, 42932, 2146, 726, 8, 1492, 612, 33564, 341, 286, 612, 726, 64308, 42932, 72362, 80941, 11327, 11048, 368, 1034, 8498, 9828, 57339, 933, 262, 557, 262, 5279, 721, 18696, 45594, 2146, 726, 11, 11460, 25, 45690, 52978, 2526, 1492, 11564, 35937, 23, 29, 341, 286, 1095, 5318, 2160, 284, 11564, 487, 4291, 36703, 35152, 19970, 368, 353, 220, 17, 317, 286, 369, 4037, 304, 11460, 341, 310, 1095, 4037, 12807, 284, 659, 198, 394, 662, 49210, 198, 394, 662, 456, 13577, 340, 394, 662, 15818, 8908, 63728, 80553, 612, 726, 64308, 29938, 50943, 88346, 2622, 310, 2160, 16209, 13577, 12807, 317, 286, 457, 286, 2160, 198, 262, 557, 262, 5279, 721, 6311, 62, 21707, 45594, 2146, 726, 11, 1495, 25, 612, 496, 8, 1492, 11564, 91344, 29, 341, 286, 443, 1115, 374, 279, 6332, 315, 279, 11418, 12496, 26, 279, 1023, 5865, 304, 1618, 198, 286, 443, 1120, 1304, 2574, 17395, 21629, 340, 286, 1095, 20791, 284, 659, 1462, 456, 84070, 42932, 545, 286, 1095, 5318, 2160, 284, 7638, 0, 15428, 286, 369, 5634, 304, 20791, 2725, 11985, 7383, 8, 341, 310, 1095, 6710, 284, 5634, 56495, 1020, 300, 2966, 1020, 300, 12807, 545, 310, 422, 1095, 4427, 13577, 8, 284, 659, 70142, 673, 83103, 8, 341, 394, 2160, 2615, 4163, 5963, 317, 394, 3136, 280, 310, 457, 310, 2160, 16209, 2146, 3867, 14784, 11473, 83103, 11, 612, 726, 70142, 1125, 286, 457, 286, 2160, 198, 262, 557, 262, 5279, 721, 6311, 45594, 2146, 726, 11, 1495, 25, 612, 496, 11, 5535, 42729, 25, 612, 45701, 53344, 496, 9414, 1492, 320, 10256, 91344, 8226, 23098, 8, 341, 286, 1095, 3361, 42932, 284, 659, 1462, 456, 84070, 42729, 42932, 545, 286, 1095, 20791, 284, 659, 1462, 456, 84070, 42932, 545, 286, 1095, 5318, 2160, 284, 7638, 0, 42001, 286, 1095, 5318, 1212, 284, 220, 15, 280, 286, 1095, 5318, 1566, 49570, 6594, 6171, 284, 220, 15, 280, 286, 6471, 341, 310, 1095, 5318, 1828, 42729, 280, 310, 1095, 5318, 1212, 22567, 284, 1212, 280, 310, 6471, 341, 394, 443, 7531, 279, 1828, 5535, 3361, 4037, 11, 422, 904, 198, 394, 1828, 42729, 284, 3361, 42932, 2725, 5791, 6615, 7383, 11, 1212, 22567, 570, 15818, 545, 394, 2489, 1828, 42729, 341, 504, 4427, 1278, 8, 591, 341, 667, 422, 5535, 42729, 8962, 2146, 1342, 12335, 5069, 368, 497, 76, 5183, 368, 2526, 341, 1014, 1464, 280, 667, 457, 667, 1212, 22567, 284, 296, 5069, 368, 489, 220, 16, 280, 504, 457, 504, 2290, 591, 1464, 345, 394, 457, 310, 457, 310, 1095, 842, 284, 1828, 42729, 4875, 8908, 7383, 19970, 1535, 765, 76, 91, 296, 5069, 5344, 310, 443, 36539, 11, 1618, 584, 733, 11, 9616, 420, 12496, 311, 721, 6311, 62, 21707, 45594, 198, 310, 369, 5634, 304, 20791, 2725, 11985, 2146, 1342, 29563, 497, 408, 2526, 341, 394, 1095, 6710, 284, 5634, 56495, 1020, 300, 2966, 1020, 300, 12807, 545, 394, 422, 1095, 4427, 13577, 8, 284, 659, 70142, 673, 83103, 8, 341, 504, 1566, 49570, 6594, 6171, 284, 220, 16, 280, 504, 2160, 2615, 4163, 5963, 317, 504, 3136, 280, 394, 457, 394, 1095, 11460, 284, 5027, 14784, 11473, 83103, 11, 612, 726, 70142, 317, 394, 1566, 49570, 6594, 6171, 284, 11460, 19970, 545, 394, 2160, 16209, 2146, 31666, 317, 310, 557, 310, 2489, 1828, 42729, 341, 394, 443, 1628, 1618, 584, 4585, 279, 3361, 4037, 198, 394, 4427, 1278, 8, 591, 341, 504, 1095, 6710, 284, 296, 5470, 2966, 545, 504, 1095, 4037, 284, 659, 64308, 29938, 40168, 11661, 9286, 947, 504, 2160, 2615, 13577, 317, 504, 1212, 284, 296, 5183, 545, 504, 1566, 49570, 6594, 6171, 284, 220, 15, 280, 394, 457, 394, 2290, 591, 1464, 345, 310, 457, 286, 557, 286, 443, 1566, 49570, 6594, 6171, 374, 1268, 1690, 11460, 3782, 505, 279, 1566, 20791, 6859, 13, 1115, 374, 1511, 198, 286, 443, 369, 26679, 45311, 11460, 11, 2533, 499, 649, 956, 11117, 4028, 320, 29092, 8, 20791, 41567, 198, 286, 320, 2171, 11, 1566, 49570, 6594, 6171, 340, 262, 557, 262, 5279, 721, 79601, 12473, 49570, 6594, 6171, 1021, 286, 612, 726, 345, 286, 11460, 25, 11564, 91344, 12803, 286, 5318, 1566, 49570, 6594, 6171, 25, 23098, 345, 262, 883, 1492, 320, 10256, 91344, 8226, 23098, 8, 341, 286, 443, 19173, 11, 279, 10687, 1405, 1057, 20791, 41567, 649, 387, 45311, 627, 286, 443, 1789, 279, 10096, 315, 26679, 45311, 11460, 11, 45311, 20791, 45473, 198, 286, 443, 374, 1193, 264, 3575, 422, 264, 6859, 430, 574, 3118, 61370, 11, 2533, 420, 649, 198, 286, 443, 3063, 311, 54477, 315, 11460, 6062, 3463, 311, 387, 15528, 627, 286, 443, 1206, 1041, 74, 7806, 3727, 1057, 2324, 2653, 555, 2737, 279, 1144, 82, 9, 27148, 81, 1734, 7727, 198, 286, 443, 5497, 13, 1115, 649, 384, 1326, 13, 5353, 2990, 77, 1, 489, 330, 330, 311, 3719, 2990, 77, 1144, 77, 23811, 286, 443, 5810, 374, 264, 4062, 323, 19108, 5155, 512, 286, 341, 310, 1095, 4037, 7037, 5823, 15005, 284, 765, 5963, 91, 341, 394, 659, 75147, 198, 504, 662, 456, 13577, 340, 504, 662, 2235, 23236, 5963, 12807, 91, 341, 667, 4037, 12807, 198, 1014, 662, 2058, 746, 1014, 662, 7430, 746, 1014, 662, 543, 23236, 5, 65, 91, 510, 65, 6, 6752, 293, 16154, 77, 518, 293, 16154, 83, 7352, 13676, 2146, 65, 1192, 504, 2820, 504, 662, 15818, 8908, 3660, 340, 310, 2670, 310, 422, 1566, 49570, 6594, 6171, 871, 220, 15, 198, 394, 1024, 4037, 7037, 5823, 15005, 2146, 31666, 14527, 9912, 19970, 368, 482, 1566, 49570, 6594, 6171, 2608, 310, 341, 394, 1418, 320, 4354, 49570, 6594, 6171, 366, 11460, 19970, 2455, 504, 1024, 4037, 7037, 5823, 15005, 2146, 31666, 14527, 9912, 19970, 368, 482, 1566, 49570, 6594, 6171, 482, 220, 16, 2608, 394, 341, 504, 1566, 49570, 6594, 6171, 1447, 220, 16, 280, 394, 457, 310, 457, 286, 457, 286, 7542, 16962, 10509, 4354, 49570, 6594, 6171, 2717, 11460, 19970, 5344, 286, 320, 31666, 11, 1566, 49570, 6594, 6171, 340, 262, 557, 262, 5279, 721, 6311, 5012, 29092, 45594, 1021, 286, 612, 726, 345, 286, 1495, 25, 612, 496, 345, 286, 5535, 42729, 25, 612, 45701, 53344, 496, 12803, 262, 883, 1492, 320, 10256, 91344, 8226, 19467, 51539, 91344, 86702, 341, 286, 1095, 320, 31666, 11, 1566, 49570, 6594, 6171, 8, 284, 659, 1462, 6311, 45594, 7383, 11, 5535, 42729, 317, 286, 422, 1566, 49570, 6594, 6171, 624, 220, 15, 341, 310, 443, 1442, 1566, 49570, 6594, 6171, 374, 7315, 11, 279, 1566, 4037, 574, 264, 3361, 4037, 323, 584, 617, 198, 310, 443, 912, 45311, 5943, 198, 310, 471, 320, 31666, 11, 19467, 487, 943, 1449, 286, 457, 286, 1095, 320, 7129, 11460, 11, 1566, 49570, 6594, 6171, 8, 4125, 310, 659, 1462, 79601, 12473, 49570, 6594, 6171, 35152, 11, 1566, 49570, 6594, 6171, 629, 286, 1095, 45311, 12807, 284, 659, 1462, 18696, 45594, 2146, 31666, 14527, 9912, 19970, 368, 482, 1566, 49570, 6594, 6171, 497, 2622, 286, 11460, 5543, 27998, 35152, 19970, 368, 482, 1566, 49570, 6594, 6171, 629, 286, 443, 5456, 25, 584, 1288, 1456, 16127, 311, 1505, 5217, 15528, 11460, 198, 286, 443, 1115, 1053, 8108, 279, 3392, 315, 2160, 1713, 3876, 994, 26679, 3543, 919, 198, 286, 443, 29734, 311, 279, 12496, 304, 459, 9191, 2373, 315, 420, 1052, 271, 286, 1095, 5318, 3543, 919, 284, 19467, 487, 943, 545, 286, 422, 45311, 12807, 2124, 15475, 368, 341, 310, 471, 320, 31666, 11, 3543, 919, 317, 286, 557, 286, 443, 1115, 374, 279, 4228, 2766, 13, 4702, 1505, 682, 3254, 11460, 430, 1212, 449, 45311, 12807, 198, 286, 443, 320, 16564, 11460, 430, 7041, 2489, 45311, 12807, 340, 286, 443, 34199, 1113, 420, 505, 279, 6471, 3770, 8779, 449, 5178, 304, 264, 4279, 1162, 627, 286, 1095, 5318, 1486, 284, 659, 198, 310, 662, 29398, 6594, 12807, 198, 310, 662, 42098, 6213, 23236, 87, 91, 865, 5470, 27586, 368, 366, 45311, 12807, 5470, 27586, 1449, 286, 1418, 1486, 366, 659, 70197, 6594, 12807, 19970, 746, 310, 1024, 659, 70197, 6594, 12807, 58, 2837, 948, 66976, 6753, 2146, 359, 29092, 12807, 340, 286, 341, 310, 3543, 919, 7175, 26674, 91615, 394, 659, 70142, 14664, 70197, 6594, 12807, 58, 2837, 948, 300, 27586, 74945, 310, 13503, 310, 1486, 1447, 220, 16, 280, 286, 557, 286, 443, 4800, 3881, 1524, 810, 65198, 5457, 13, 2468, 1475, 320, 1605, 8, 3284, 2361, 369, 279, 610, 723, 2785, 198, 286, 443, 4037, 11, 78884, 5217, 5943, 505, 430, 4037, 320, 333, 904, 8, 311, 45311, 12807, 345, 286, 443, 323, 2160, 1713, 1082, 279, 4459, 3245, 323, 1518, 1148, 584, 636, 627, 286, 369, 602, 304, 220, 16, 497, 359, 29092, 12807, 19970, 368, 341, 310, 1095, 9436, 284, 612, 359, 29092, 12807, 96974, 72, 947, 310, 1095, 21166, 284, 612, 359, 29092, 12807, 1004, 497, 947, 310, 1095, 5318, 1486, 284, 659, 198, 394, 662, 29398, 6594, 12807, 198, 394, 662, 42098, 6213, 23236, 87, 91, 865, 5470, 27586, 368, 366, 21166, 317, 310, 443, 5456, 25, 72548, 7706, 8082, 422, 21166, 8638, 449, 330, 330, 5380, 310, 1418, 1486, 366, 659, 70197, 6594, 12807, 19970, 746, 394, 1024, 659, 70197, 6594, 12807, 58, 2837, 948, 66976, 6753, 97566, 340, 310, 341, 394, 1095, 13336, 284, 510, 12113, 11, 659, 70197, 6594, 12807, 58, 2837, 948, 300, 27586, 58690, 20773, 545, 394, 1095, 21136, 284, 2489, 1487, 487, 496, 487, 1527, 40553, 23, 2146, 9007, 3225, 8, 341, 504, 443, 8613, 750, 11, 420, 374, 5027, 14784, 11473, 2146, 9007, 3225, 11, 612, 726, 70142, 340, 504, 443, 2030, 584, 2643, 617, 11784, 264, 20791, 6859, 902, 1053, 5471, 82053, 627, 504, 443, 320, 74039, 3284, 304, 279, 9546, 315, 45311, 20791, 41567, 340, 504, 443, 2100, 5625, 311, 20677, 12, 23, 323, 656, 20791, 45473, 627, 504, 443, 469, 1326, 13, 449, 1206, 1041, 74, 7806, 330, 220, 97186, 5334, 6859, 311, 330, 330, 489, 330, 758, 761, 504, 443, 719, 5027, 14784, 11473, 446, 220, 758, 909, 976, 5027, 14784, 11473, 446, 14501, 504, 7777, 1161, 8, 591, 659, 1462, 6311, 62, 21707, 45594, 1161, 18966, 504, 443, 7146, 2740, 11, 3508, 477, 539, 420, 6916, 374, 4495, 14117, 389, 3508, 1070, 198, 504, 443, 1053, 387, 264, 20791, 6859, 1603, 279, 20677, 12, 23, 63950, 367, 1486, 627, 504, 443, 38254, 35149, 3403, 430, 912, 832, 690, 3596, 5406, 320, 10924, 682, 11, 1274, 3287, 956, 198, 504, 443, 5406, 682, 279, 2466, 20349, 304, 279, 3766, 45311, 4037, 8292, 340, 504, 15863, 49239, 591, 5027, 14784, 11473, 2146, 9007, 3225, 11, 612, 726, 70142, 1350, 504, 443, 25681, 1093, 279, 2768, 374, 41765, 719, 15465, 512, 504, 443, 15863, 2069, 8, 591, 659, 1462, 6311, 62, 21707, 45594, 7, 39257, 341, 504, 443, 257, 1487, 487, 496, 487, 1527, 40553, 23, 5012, 7702, 2146, 9007, 3225, 96974, 68, 21232, 8401, 2401, 57124, 504, 443, 12240, 394, 2670, 394, 1095, 5318, 13278, 284, 11564, 487, 943, 545, 394, 1095, 5318, 13278, 6171, 284, 220, 15, 280, 394, 369, 4037, 304, 21136, 341, 504, 13278, 2615, 13577, 317, 504, 13278, 6171, 1447, 659, 75147, 58, 5, 5963, 948, 2963, 545, 504, 422, 13278, 6171, 2669, 45311, 12807, 19970, 368, 341, 667, 1464, 280, 504, 457, 394, 457, 394, 3543, 919, 7175, 35920, 317, 394, 1486, 1447, 220, 16, 280, 310, 457, 286, 557, 286, 443, 1115, 374, 1101, 539, 31439, 13, 6104, 584, 8965, 9855, 430, 20791, 41567, 527, 15528, 345, 286, 443, 26907, 11, 814, 527, 539, 13, 3011, 374, 11, 422, 7999, 5943, 1051, 311, 1304, 264, 6859, 5101, 304, 198, 286, 443, 45311, 12807, 11, 420, 1436, 1304, 11460, 3284, 902, 1057, 12496, 1053, 6062, 1781, 198, 286, 443, 1053, 387, 27092, 627, 286, 443, 1789, 3187, 11, 449, 342, 418, 17, 11, 279, 1005, 315, 1144, 82, 13666, 30, 15114, 50, 8, 3445, 430, 2990, 77, 1734, 1, 1436, 198, 286, 443, 2274, 264, 6859, 11, 384, 1326, 13, 2990, 77, 1734, 15, 1, 41567, 1139, 2990, 77, 1, 37369, 77, 78828, 15, 498, 3339, 2990, 77, 1, 264, 3284, 4037, 627, 286, 443, 5810, 374, 264, 4062, 323, 19108, 5155, 512, 286, 443, 1115, 4536, 956, 1314, 422, 584, 3596, 4148, 1144, 82, 13666, 30, 15114, 50, 340, 286, 422, 45311, 12807, 19970, 368, 871, 220, 16, 341, 310, 1095, 1566, 14102, 6879, 284, 293, 496, 487, 18696, 12473, 40553, 23, 18870, 29092, 12807, 5470, 27586, 1449, 310, 422, 45311, 12807, 19970, 368, 482, 1566, 14102, 6879, 13, 16, 871, 220, 15, 198, 394, 1024, 1566, 14102, 6879, 13, 15, 4875, 8908, 3660, 11, 765, 66, 91, 272, 2124, 87275, 2455, 310, 341, 394, 1095, 5318, 312, 19889, 284, 5027, 14784, 11473, 1021, 504, 612, 359, 29092, 12807, 96974, 359, 29092, 12807, 19970, 368, 482, 1566, 14102, 6879, 13, 16, 1282, 504, 612, 726, 70142, 345, 394, 1465, 394, 312, 19889, 16209, 20318, 14784, 11473, 1021, 504, 612, 359, 29092, 12807, 58, 359, 29092, 12807, 19970, 368, 482, 1566, 14102, 6879, 13, 16, 497, 1282, 504, 612, 726, 70142, 345, 394, 16925, 394, 3543, 919, 7175, 5921, 19889, 317, 310, 457, 286, 557, 286, 320, 31666, 11, 3543, 919, 340, 262, 457, 633, 13657, 79, 1631, 1086, 82, 933, 6517, 9708, 33, 1777, 341, 262, 11765, 943, 933, 262, 5279, 502, 1021, 286, 24592, 25, 10751, 51539, 35937, 23, 8226, 23098, 12803, 286, 3361, 29938, 40168, 25, 10751, 3548, 11, 23098, 12803, 286, 5497, 25, 612, 496, 345, 262, 883, 1492, 5468, 2122, 62848, 29, 341, 286, 1095, 20791, 284, 27238, 487, 943, 32048, 340, 310, 662, 2235, 9450, 23236, 68, 91, 76767, 487, 943, 28738, 62852, 487, 14149, 1150, 1480, 11, 721, 2284, 68, 2446, 3991, 10340, 70593, 286, 1095, 3361, 42932, 284, 341, 310, 1095, 721, 18753, 284, 3361, 29938, 40168, 198, 394, 662, 10786, 746, 394, 662, 2235, 23236, 82, 91, 27555, 42932, 487, 13295, 1161, 1192, 394, 662, 17840, 28738, 10256, 33499, 38138, 310, 27238, 487, 943, 50138, 18753, 5563, 39647, 5572, 394, 662, 2235, 9450, 23236, 68, 91, 76767, 487, 943, 28738, 62852, 487, 14149, 1150, 1480, 11, 721, 2284, 68, 2446, 3991, 10340, 5380, 286, 3718, 286, 1095, 25569, 25, 10751, 91344, 11, 11564, 35937, 23, 2511, 4125, 310, 24592, 20036, 1020, 2235, 23236, 7, 74, 11, 348, 18419, 4712, 85, 11, 597, 16380, 35770, 17840, 1454, 286, 2105, 10509, 28106, 19970, 368, 624, 25569, 19970, 5344, 286, 1095, 3361, 29938, 50943, 25, 10751, 91344, 11, 11564, 35937, 23, 2511, 284, 3361, 29938, 40168, 198, 310, 662, 2058, 746, 310, 662, 2235, 23236, 7, 74, 11, 348, 18419, 4712, 85, 11, 597, 5470, 12807, 1020, 998, 13554, 12419, 310, 662, 17840, 1454, 286, 443, 29013, 1606, 358, 1541, 956, 1440, 1268, 311, 3371, 34889, 358, 2846, 539, 2133, 311, 2349, 279, 2472, 198, 286, 1095, 5318, 10839, 6594, 12807, 25, 11564, 51539, 35937, 23, 2511, 284, 24592, 9301, 1020, 566, 20262, 1020, 17840, 545, 286, 10839, 6594, 12807, 10838, 1454, 286, 7777, 78720, 33, 1777, 341, 310, 24592, 345, 310, 3361, 29938, 40168, 345, 310, 25569, 345, 310, 3361, 29938, 50943, 345, 310, 20791, 72362, 25, 320, 15, 497, 10809, 9828, 57339, 570, 2235, 23236, 36495, 20791, 16380, 6139, 17840, 3227, 310, 3361, 42932, 72362, 25, 320, 15, 497, 10809, 9828, 57339, 340, 394, 662, 2235, 23236, 36495, 3361, 42932, 16380, 2455, 394, 662, 17840, 3227, 310, 10839, 6594, 12807, 345, 286, 2820, 262, 557, 262, 443, 25855, 20168, 262, 443, 30430, 198, 262, 443, 25855, 8880, 271, 262, 5279, 16559, 62, 21707, 2146, 726, 11, 4611, 25, 13325, 11, 1495, 25, 612, 496, 8, 1492, 11564, 91344, 29, 341, 286, 4611, 58556, 30825, 80553, 659, 1462, 6311, 62, 21707, 45594, 7383, 1192, 262, 557, 262, 5279, 16559, 2146, 726, 11, 4611, 25, 13325, 11, 1495, 25, 612, 496, 11, 5535, 42729, 25, 19467, 53344, 496, 9414, 1492, 11564, 91344, 29, 341, 286, 4611, 58556, 30825, 80553, 659, 1462, 6311, 45594, 7383, 11, 612, 21642, 42729, 570, 15, 340, 262, 557, 262, 5279, 721, 6311, 12807, 2146, 726, 11, 4611, 25, 13325, 11, 5943, 25, 45690, 84, 23, 2526, 1492, 11564, 91344, 29, 341, 286, 4611, 58556, 30825, 80553, 341, 310, 2489, 1487, 487, 496, 487, 1527, 40553, 23, 24033, 8, 341, 394, 7777, 7383, 8, 591, 659, 1462, 6311, 62, 21707, 45594, 7383, 1350, 394, 15863, 2069, 8, 591, 341, 504, 1095, 1495, 284, 20451, 314, 1487, 487, 496, 487, 1527, 40553, 23, 5012, 7702, 2146, 9848, 96974, 68, 21232, 8401, 2401, 368, 2526, 2670, 504, 1095, 320, 31666, 11, 1566, 49570, 6594, 6171, 8, 284, 659, 1462, 6311, 45594, 7383, 11, 612, 45701, 487, 943, 1449, 504, 1095, 320, 7129, 11460, 11, 1566, 49570, 6594, 6171, 8, 4125, 667, 659, 1462, 79601, 12473, 49570, 6594, 6171, 35152, 11, 1566, 49570, 6594, 6171, 317, 504, 422, 758, 31666, 2124, 15475, 368, 1024, 1566, 49570, 6594, 6171, 871, 220, 15, 341, 667, 443, 445, 454, 1022, 279, 11460, 505, 279, 1566, 6710, 323, 1629, 426, 1777, 389, 279, 9861, 5943, 198, 667, 443, 18024, 81575, 35149, 11, 719, 420, 1253, 539, 387, 4495, 422, 584, 4265, 617, 1047, 264, 20791, 198, 667, 443, 6859, 1990, 279, 2764, 20677, 12, 23, 323, 279, 8482, 5943, 11, 902, 374, 3249, 420, 198, 667, 443, 1749, 374, 879, 198, 667, 1095, 5318, 45311, 12807, 4125, 1014, 659, 1462, 18696, 45594, 2146, 31666, 14527, 9912, 19970, 368, 482, 1566, 49570, 6594, 6171, 497, 2622, 667, 45311, 12807, 16209, 5791, 27586, 2146, 9848, 23876, 21232, 8401, 2401, 368, 497, 10359, 667, 11460, 5543, 27998, 35152, 19970, 368, 482, 1566, 49570, 6594, 6171, 317, 667, 11460, 16209, 20318, 14784, 11473, 2146, 359, 29092, 12807, 11, 612, 726, 70142, 1125, 504, 457, 504, 11460, 198, 394, 457, 310, 457, 286, 2820, 262, 557, 262, 5279, 16559, 6753, 5012, 29092, 1021, 286, 612, 726, 345, 286, 4611, 25, 13325, 345, 286, 1495, 25, 612, 496, 345, 286, 5535, 42729, 25, 19467, 53344, 496, 12803, 262, 883, 1492, 5468, 27, 14149, 29781, 29, 341, 286, 1095, 320, 31666, 11, 3543, 919, 8, 4125, 310, 4611, 58556, 30825, 80553, 659, 1462, 6311, 5012, 29092, 45594, 7383, 11, 612, 21642, 42729, 1125, 286, 1095, 4611, 3038, 11053, 919, 4125, 310, 5468, 861, 487, 943, 47927, 11, 3543, 919, 20036, 1020, 2235, 23236, 13565, 91, 5468, 861, 487, 943, 47927, 11, 612, 13565, 96974, 82968, 286, 320, 31666, 11, 4611, 3038, 11053, 919, 570, 18614, 41391, 47927, 340, 262, 557, 262, 5279, 16559, 20052, 6594, 2146, 726, 11, 6710, 25, 45690, 84, 23, 2526, 1492, 5468, 2122, 91344, 29, 341, 286, 422, 1095, 4427, 13577, 8, 284, 659, 70142, 673, 83103, 570, 38828, 1142, 368, 341, 310, 471, 7777, 13577, 317, 286, 457, 286, 422, 1095, 7777, 83103, 2966, 8, 284, 1487, 487, 496, 487, 1527, 40553, 23, 83103, 8, 341, 310, 422, 1095, 4427, 13577, 8, 284, 659, 64308, 29938, 40168, 673, 83103, 2966, 570, 38828, 1142, 368, 341, 394, 471, 7777, 13577, 317, 310, 457, 286, 457, 286, 15863, 43098, 7903, 487, 943, 28738, 62852, 487, 14149, 1622, 1480, 11, 721, 2284, 23164, 2446, 53073, 12419, 262, 557, 262, 5279, 16559, 20052, 49570, 2146, 726, 11, 6710, 25, 45690, 84, 23, 2526, 1492, 11564, 91344, 29, 341, 286, 422, 1095, 4427, 13577, 8, 284, 659, 70142, 673, 83103, 8, 341, 310, 471, 7638, 21362, 9, 5963, 947, 286, 457, 286, 5027, 14784, 11473, 83103, 11, 612, 726, 70142, 340, 262, 557, 262, 443, 25855, 20168, 262, 443, 3799, 3785, 198, 262, 443, 25855, 8880, 271, 262, 5279, 17322, 12807, 2146, 726, 11, 4611, 25, 13325, 11, 11460, 25, 11564, 91344, 9414, 1492, 5468, 27, 14149, 7223, 29, 341, 286, 1095, 5943, 284, 4611, 58556, 30825, 80553, 659, 1462, 18696, 45594, 2146, 31666, 1125, 286, 5468, 7223, 487, 943, 47927, 11, 612, 9848, 570, 18614, 746, 262, 557, 262, 5279, 17322, 20052, 6594, 12807, 2146, 726, 11, 4611, 25, 13325, 11, 4037, 25, 23098, 8, 1492, 5468, 2122, 27, 14149, 27, 14149, 7223, 2511, 341, 286, 422, 1095, 4427, 24033, 8, 284, 659, 75147, 673, 2146, 5963, 8, 341, 310, 471, 7777, 43098, 7223, 487, 943, 47927, 11, 5943, 570, 18614, 1449, 286, 457, 286, 422, 1095, 4427, 24033, 8, 284, 659, 64308, 29938, 50943, 673, 2146, 5963, 8, 341, 310, 471, 7777, 43098, 7223, 487, 943, 47927, 11, 5943, 570, 18614, 1449, 286, 457, 286, 15863, 43098, 7903, 487, 943, 28738, 62852, 487, 14149, 1622, 1480, 11, 721, 2284, 5963, 2446, 3991, 12419, 262, 557, 262, 443, 25855, 20168, 262, 443, 79377, 198, 262, 443, 25855, 8880, 271, 262, 5279, 4037, 20317, 9324, 2146, 726, 11, 4611, 25, 13325, 8, 1492, 11564, 27, 14149, 27, 14149, 7223, 2511, 341, 286, 659, 70197, 6594, 12807, 198, 310, 662, 2058, 746, 310, 662, 2235, 23236, 87, 91, 5468, 7223, 487, 943, 47927, 11, 865, 570, 18614, 2455, 310, 662, 17840, 746, 262, 457, 633, 13657, 79, 1631, 1793, 933, 8998, 721, 83, 1609, 5963, 2551, 3368, 25, 13325, 11, 296, 25, 612, 14149, 3413, 8, 1492, 5468, 2122, 72798, 341, 262, 296, 1388, 4895, 28738, 5501, 33, 1777, 13867, 38545, 262, 7777, 51532, 633, 13657, 14401, 8793, 5680, 2658, 7177, 341, 262, 1005, 23941, 66, 9127, 487, 82956, 19004, 439, 10751, 401, 262, 1005, 18187, 487, 3867, 14784, 17489, 401, 262, 11765, 1985, 933, 262, 5279, 1633, 31115, 4552, 368, 341, 286, 1095, 5318, 21467, 284, 10751, 487, 2309, 545, 286, 21467, 7175, 1921, 1, 370, 3343, 998, 13554, 1535, 220, 16, 317, 286, 21467, 7175, 1921, 1, 4484, 3343, 998, 13554, 1535, 220, 17, 629, 286, 1095, 594, 284, 5027, 14784, 17489, 1921, 1, 69744, 498, 612, 81, 4129, 317, 286, 2105, 10945, 10509, 417, 11, 7638, 21362, 65, 1, 370, 498, 293, 1, 4484, 15399, 262, 457, 534] \ No newline at end of file diff --git a/test/Microsoft.ML.Tokenizers.Tests/Data/tokens_gpt2.json b/test/Microsoft.ML.Tokenizers.Tests/Data/tokens_gpt2.json new file mode 100644 index 0000000000..e389642d70 --- /dev/null +++ b/test/Microsoft.ML.Tokenizers.Tests/Data/tokens_gpt2.json @@ -0,0 +1 @@ +[1003, 770, 2198, 318, 649, 290, 2331, 46542, 357, 39363, 351, 9485, 46, 18, 10375, 8, 198, 2, 0, 58, 12154, 7, 565, 41214, 3712, 2865, 808, 62, 67, 567, 69, 62, 5420, 15437, 198, 198, 1904, 14367, 3712, 4033, 26448, 3712, 26257, 7248, 26, 198, 1904, 14367, 3712, 16663, 26, 198, 198, 1904, 14996, 62, 260, 25636, 3712, 3041, 25636, 26, 198, 1904, 279, 8226, 18, 3712, 1069, 11755, 26, 198, 1904, 279, 8226, 18, 3712, 79, 2411, 2507, 3712, 9, 26, 198, 1904, 279, 8226, 18, 3712, 19199, 3712, 90, 20519, 45992, 11, 9485, 8053, 11, 9485, 51, 29291, 19629, 198, 1904, 279, 8226, 18, 3712, 20519, 23004, 26, 198, 1904, 17000, 66, 62, 17831, 3712, 37, 87, 26257, 13912, 355, 21059, 13912, 26, 198, 198, 22184, 4808, 26327, 62, 24874, 62, 647, 469, 27, 51, 33994, 198, 220, 220, 220, 3704, 25, 1222, 58, 84, 23, 4357, 198, 220, 220, 220, 9803, 25, 1222, 26257, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 22330, 198, 220, 220, 220, 277, 25, 4114, 37481, 7, 19282, 3712, 2840, 3712, 17257, 27, 385, 1096, 43734, 4613, 309, 11, 198, 8, 4613, 38692, 27, 51, 29, 1391, 198, 220, 220, 220, 3373, 770, 318, 257, 15879, 286, 357, 9688, 11, 4279, 737, 198, 220, 220, 220, 3373, 383, 4279, 318, 286, 262, 18022, 5166, 3599, 379, 2292, 923, 13, 198, 220, 220, 220, 3373, 383, 4279, 286, 262, 938, 2378, 287, 262, 15879, 318, 407, 257, 4938, 1988, 13, 198, 220, 220, 220, 1309, 4517, 3354, 25, 38692, 27, 7, 385, 1096, 11, 514, 1096, 8, 29, 796, 357, 15, 492, 12239, 13, 11925, 3419, 1343, 352, 737, 8899, 7, 91, 72, 91, 357, 72, 11, 514, 1096, 3712, 22921, 29720, 33327, 9783, 628, 220, 220, 220, 1309, 651, 62, 43027, 796, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1303, 58, 45145, 7, 33770, 15437, 198, 220, 220, 220, 220, 220, 220, 220, 930, 42632, 25, 1222, 53, 721, 27, 7, 385, 1096, 11, 514, 1096, 8, 22330, 923, 62, 312, 87, 25, 514, 1096, 11, 14267, 25, 514, 1096, 91, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 357, 9688, 62, 312, 87, 1343, 14267, 1343, 362, 8, 1279, 3354, 13, 11925, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 9803, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 1136, 39434, 12239, 58, 42632, 58, 9688, 62, 312, 87, 4083, 15, 492, 42632, 58, 9688, 62, 312, 87, 1343, 14267, 1343, 362, 4083, 15, 12962, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 22163, 798, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 2073, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 6045, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 18083, 628, 220, 220, 220, 3373, 775, 804, 510, 262, 9803, 1752, 287, 262, 3726, 290, 11629, 9404, 4296, 198, 220, 220, 220, 3373, 606, 1141, 1123, 20121, 11, 543, 12850, 262, 1271, 286, 4279, 804, 4739, 13, 198, 220, 220, 220, 329, 1312, 287, 657, 492, 42632, 13, 11925, 3419, 532, 362, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 2872, 651, 62, 43027, 39434, 42632, 11, 1312, 11, 657, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2773, 7, 43027, 8, 5218, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 514, 1096, 3712, 22921, 318, 257, 1908, 20538, 1988, 290, 2314, 307, 257, 4938, 4279, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 14257, 62, 30493, 0, 7, 43027, 14512, 514, 1096, 3712, 22921, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3354, 58, 72, 4083, 16, 796, 4279, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 6045, 5218, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2555, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 18083, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 3373, 1002, 345, 423, 299, 3354, 290, 285, 4017, 3212, 11, 428, 857, 440, 7, 10295, 8, 670, 13, 198, 220, 220, 220, 3373, 775, 714, 466, 1223, 351, 257, 24575, 290, 466, 440, 7, 76, 2604, 299, 8, 670, 13, 198, 220, 220, 220, 3373, 632, 318, 1593, 284, 2074, 326, 299, 318, 1690, 1402, 38155, 3064, 828, 290, 355, 884, 198, 220, 220, 220, 3373, 262, 12940, 12, 17946, 1483, 4034, 31405, 262, 8385, 9383, 13357, 21838, 1460, 198, 220, 220, 220, 3373, 286, 262, 4600, 42632, 63, 15879, 1366, 4645, 2029, 13, 628, 220, 220, 220, 3373, 5740, 326, 356, 12234, 9881, 11, 407, 11241, 14729, 13, 1081, 890, 355, 356, 4512, 347, 11401, 262, 835, 356, 198, 220, 220, 220, 3373, 3058, 466, 11, 428, 318, 7548, 13, 1052, 2562, 835, 284, 2270, 428, 561, 307, 284, 875, 43846, 198, 220, 220, 220, 3373, 20121, 8475, 422, 11241, 6376, 393, 284, 2948, 2176, 11241, 4017, 3212, 13, 198, 220, 220, 220, 9052, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 611, 3354, 13, 11925, 3419, 6624, 352, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2270, 26, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 3373, 514, 1096, 3712, 22921, 318, 257, 1908, 20538, 4279, 1988, 5086, 514, 284, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 1011, 262, 949, 517, 2952, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 949, 62, 43027, 25, 357, 385, 1096, 11, 514, 1096, 8, 796, 357, 385, 1096, 3712, 22921, 11, 657, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 329, 357, 72, 11, 1222, 28264, 11, 4279, 4008, 287, 3354, 58, 492, 42632, 13, 11925, 3419, 532, 352, 4083, 2676, 22446, 268, 6975, 378, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 4279, 1279, 949, 62, 43027, 13, 15, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 949, 62, 43027, 796, 357, 43027, 11, 1312, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 611, 949, 62, 43027, 13, 15, 14512, 514, 1096, 3712, 22921, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 1312, 796, 949, 62, 43027, 13, 16, 26, 628, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 24550, 25, 775, 389, 546, 284, 4781, 3354, 58, 72, 1343, 352, 4083, 775, 466, 407, 466, 340, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 1865, 780, 612, 389, 12940, 12, 17946, 1483, 4034, 284, 19698, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 3354, 58, 72, 60, 290, 3354, 58, 72, 12, 16, 60, 878, 10829, 11, 543, 714, 5636, 1077, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 262, 12940, 13, 6660, 11, 356, 4296, 262, 4279, 17952, 416, 31017, 625, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 3354, 58, 72, 1343, 352, 4357, 416, 39744, 4600, 1136, 62, 43027, 0, 63, 351, 4600, 48267, 796, 352, 44646, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3354, 58, 72, 4083, 16, 796, 651, 62, 43027, 39434, 42632, 11, 1312, 11, 352, 737, 403, 37150, 62, 273, 7, 385, 1096, 3712, 22921, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 1312, 1875, 657, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3354, 58, 72, 532, 352, 4083, 16, 796, 651, 62, 43027, 39434, 42632, 11, 1312, 532, 352, 11, 352, 737, 403, 37150, 62, 273, 7, 385, 1096, 3712, 22921, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3354, 13, 28956, 7, 72, 1343, 352, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 2073, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2270, 26, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 1782, 198, 220, 220, 220, 1309, 4517, 503, 25, 38692, 27, 51, 29, 796, 38692, 3712, 4480, 62, 42404, 7, 42632, 13, 11925, 3419, 532, 352, 1776, 198, 220, 220, 220, 329, 1312, 287, 657, 492, 42632, 13, 11925, 3419, 532, 352, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 503, 13, 14689, 7, 69, 7, 42632, 58, 72, 4083, 15, 492, 42632, 58, 72, 1343, 352, 4083, 15, 18125, 198, 220, 220, 220, 1782, 198, 220, 220, 220, 503, 198, 92, 198, 198, 12984, 24714, 18022, 62, 24874, 62, 268, 8189, 7, 12239, 25, 1222, 58, 84, 23, 4357, 9803, 25, 1222, 26257, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 43734, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 220, 220, 220, 611, 3704, 13, 11925, 3419, 6624, 352, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1441, 43030, 0, 58, 81, 2283, 58, 12239, 60, 11208, 198, 220, 220, 220, 1782, 198, 220, 220, 220, 4808, 26327, 62, 24874, 62, 647, 469, 7, 12239, 11, 9803, 11, 930, 79, 91, 9803, 58, 5, 12239, 58, 79, 13, 9688, 492, 79, 13, 437, 11907, 8, 198, 92, 198, 198, 12984, 24714, 18022, 62, 24874, 62, 35312, 27, 6, 64, 33994, 12239, 25, 1222, 6, 64, 685, 84, 23, 4357, 9803, 25, 1222, 26257, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 43734, 4613, 38692, 27, 5, 6, 64, 685, 84, 23, 60, 29, 1391, 198, 220, 220, 220, 611, 3704, 13, 11925, 3419, 6624, 352, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1441, 43030, 0, 58, 12239, 11208, 198, 220, 220, 220, 1782, 198, 220, 220, 220, 4808, 26327, 62, 24874, 62, 647, 469, 7, 12239, 11, 9803, 11, 930, 79, 91, 1222, 12239, 58, 79, 13, 9688, 492, 79, 13, 437, 12962, 198, 92, 198, 198, 1003, 26386, 2854, 4710, 25, 198, 1003, 198, 1003, 797, 25636, 198, 1003, 29335, 198, 1003, 4042, 286, 262, 640, 318, 3377, 287, 40364, 13, 383, 16638, 835, 284, 2866, 428, 510, 318, 416, 1262, 1342, 14996, 198, 1003, 40364, 3033, 13, 1114, 4554, 11, 1262, 257, 40364, 21136, 12, 540, 416, 4600, 260, 25636, 63, 27021, 318, 513, 87, 5443, 621, 198, 1003, 262, 6678, 40364, 356, 779, 13, 198, 1003, 198, 1003, 2102, 11, 1813, 326, 356, 821, 1262, 257, 40364, 21136, 12, 540, 416, 4600, 260, 25636, 47671, 612, 2125, 470, 881, 3580, 198, 1003, 1022, 1262, 262, 4600, 260, 25636, 63, 27021, 290, 1262, 262, 4600, 69, 3883, 62, 260, 25636, 63, 27021, 13, 198, 1003, 198, 1003, 1318, 318, 281, 1593, 10375, 1022, 4704, 278, 11, 4600, 260, 25636, 63, 290, 4600, 69, 3883, 62, 260, 25636, 44646, 198, 1003, 1649, 1262, 4600, 69, 3883, 62, 260, 25636, 47671, 356, 2277, 4600, 260, 25636, 13, 19796, 62, 265, 44646, 632, 4962, 503, 326, 428, 5640, 22627, 319, 198, 1003, 617, 4517, 540, 12692, 2272, 2641, 286, 4600, 260, 25636, 44646, 770, 5543, 12847, 2854, 13, 1649, 1262, 8631, 198, 1003, 1468, 4600, 260, 25636, 47671, 356, 836, 470, 2277, 428, 11, 780, 4600, 19796, 62, 2676, 63, 468, 257, 1180, 2438, 3108, 13, 198, 1003, 19809, 25, 3740, 1378, 12567, 13, 785, 14, 11469, 12, 17204, 14, 260, 25636, 14, 2436, 672, 14, 9866, 14, 18973, 13775, 10725, 5222, 13, 9132, 198, 1003, 21836, 11, 262, 835, 356, 651, 1088, 428, 318, 351, 1719, 257, 357, 29471, 8, 4704, 1957, 17271, 286, 262, 40364, 329, 198, 1003, 1123, 4704, 13, 198, 1003, 198, 1003, 14122, 278, 198, 1003, 796, 2559, 198, 1003, 314, 3088, 1262, 4600, 2433, 261, 44646, 632, 2492, 470, 1107, 5443, 621, 1262, 11361, 14390, 290, 13011, 262, 402, 4146, 13, 198, 1003, 1406, 24829, 4600, 2433, 261, 63, 0, 3914, 4704, 954, 3503, 307, 287, 1630, 286, 674, 11361, 2985, 13, 198, 1003, 198, 1003, 327, 8103, 198, 1003, 29335, 855, 198, 1003, 383, 4941, 11241, 5847, 468, 281, 300, 622, 12940, 625, 262, 7548, 286, 4600, 26327, 62, 24874, 62, 268, 8189, 44646, 198, 1003, 19486, 11, 356, 550, 530, 1165, 0, 9170, 340, 11, 356, 547, 691, 29627, 5443, 621, 11361, 13, 198, 1003, 314, 973, 281, 33212, 25392, 284, 1805, 262, 12940, 13, 770, 1422, 470, 1283, 284, 5938, 2060, 40945, 2854, 198, 1003, 33752, 11, 475, 340, 750, 2689, 5021, 12, 16663, 276, 2854, 13, 34097, 306, 11, 340, 3947, 284, 2689, 198, 1003, 5021, 12, 16663, 276, 2854, 772, 618, 314, 691, 550, 7183, 357, 11261, 3077, 314, 32621, 1223, 510, 29865, 198, 1003, 21836, 11, 314, 19169, 326, 356, 714, 651, 5755, 286, 262, 12940, 11, 611, 356, 2190, 262, 900, 286, 16326, 355, 257, 12940, 0, 198, 1003, 2312, 389, 3446, 262, 900, 393, 4017, 3212, 326, 389, 1884, 284, 307, 3024, 13, 843, 783, 356, 836, 470, 423, 284, 892, 198, 1003, 546, 11087, 4517, 1799, 11, 4088, 779, 11, 393, 45973, 13, 198, 1003, 198, 1003, 367, 2140, 198, 1003, 29335, 855, 198, 1003, 775, 779, 376, 87, 26257, 13912, 2427, 286, 262, 3210, 21059, 13912, 13, 770, 318, 3863, 588, 257, 642, 12, 940, 4, 1592, 30, 198, 1003, 383, 1459, 7822, 5645, 510, 1804, 257, 1256, 286, 49544, 286, 9881, 13, 554, 4583, 11, 428, 714, 307, 925, 198, 1003, 284, 307, 49544, 286, 734, 12, 28047, 2374, 286, 493, 82, 11, 543, 3073, 588, 340, 743, 635, 307, 257, 3155, 1411, 5443, 13, 198, 198, 1904, 14367, 3712, 22510, 3712, 15419, 28667, 52, 2414, 26, 198, 12984, 2878, 33482, 16818, 7390, 7, 15419, 28667, 52, 2414, 1776, 198, 198, 22184, 12234, 62, 14421, 62, 16663, 3419, 4613, 514, 1096, 1391, 198, 220, 220, 220, 3373, 632, 338, 4577, 284, 779, 21596, 621, 284, 779, 37862, 13, 17103, 468, 428, 3621, 334, 2414, 4704, 4686, 3753, 198, 220, 220, 220, 3373, 326, 2499, 1049, 329, 674, 779, 1339, 286, 14928, 31998, 287, 674, 7177, 13, 8989, 11, 198, 220, 220, 220, 3373, 340, 338, 2839, 13, 2102, 11, 612, 389, 691, 523, 867, 2842, 345, 460, 12461, 257, 334, 2414, 11, 523, 655, 21595, 1133, 198, 220, 220, 220, 3373, 3740, 1378, 12567, 13, 785, 14, 11469, 12, 17204, 14, 11469, 14, 37165, 14, 37601, 2670, 198, 220, 220, 220, 1500, 4808, 25, 685, 84, 23, 26, 807, 60, 796, 685, 15, 26, 14367, 3712, 11883, 3712, 7857, 62, 1659, 3712, 27, 19282, 3712, 16663, 3712, 16818, 7390, 29, 3419, 11208, 198, 220, 220, 220, 1500, 4808, 25, 685, 84, 23, 26, 807, 60, 796, 685, 15, 26, 14367, 3712, 11883, 3712, 7857, 62, 1659, 3712, 27, 49233, 16818, 7390, 29, 3419, 11208, 198, 220, 220, 220, 1309, 2124, 796, 21596, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 14367, 3712, 11883, 3712, 7645, 76, 1133, 3712, 27, 19282, 3712, 16663, 3712, 16818, 7390, 11, 33482, 16818, 7390, 33994, 16663, 3712, 14421, 22446, 312, 3419, 737, 15, 198, 220, 220, 220, 18083, 198, 220, 220, 220, 334, 2414, 3712, 6738, 7, 87, 8, 355, 514, 1096, 198, 92, 198, 198, 9979, 25882, 62, 41359, 62, 4221, 15675, 50, 25, 514, 1096, 796, 13108, 26, 198, 2, 58, 9078, 4871, 60, 198, 7249, 7231, 33, 11401, 1391, 198, 220, 220, 220, 2207, 12342, 25, 21059, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 22330, 198, 220, 220, 220, 2041, 62, 83, 482, 641, 62, 12685, 12342, 25, 21059, 13912, 27, 10100, 11, 514, 1096, 22330, 198, 220, 220, 220, 875, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 11, 198, 220, 220, 220, 2041, 62, 83, 482, 641, 62, 12501, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 11, 198, 220, 220, 220, 40364, 62, 83, 7278, 25, 38692, 27, 3041, 25636, 22330, 198, 220, 220, 220, 2041, 62, 260, 25636, 62, 83, 7278, 25, 38692, 27, 3041, 25636, 22330, 198, 220, 220, 220, 23243, 62, 30001, 62, 33661, 25, 38692, 27, 53, 721, 27, 84, 23, 4211, 11, 198, 92, 198, 198, 23928, 7231, 33, 11401, 1391, 198, 220, 220, 220, 24714, 4808, 1136, 62, 28781, 62, 260, 25636, 39434, 944, 8, 4613, 1222, 3041, 25636, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 4091, 2854, 4710, 2029, 329, 644, 428, 318, 546, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 632, 338, 635, 257, 1310, 474, 39556, 11, 3387, 787, 257, 1365, 2196, 286, 340, 0, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 2102, 11, 340, 338, 3621, 326, 428, 1595, 470, 13044, 4088, 284, 1790, 12, 24489, 14390, 198, 220, 220, 220, 220, 220, 220, 220, 1222, 944, 13, 260, 25636, 62, 83, 7278, 58, 17831, 62, 14421, 62, 16663, 3419, 4064, 25882, 62, 41359, 62, 4221, 15675, 50, 60, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 4808, 1136, 62, 28781, 62, 20887, 62, 260, 25636, 39434, 944, 8, 4613, 1222, 3041, 25636, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1222, 944, 13, 20887, 62, 260, 25636, 62, 83, 7278, 58, 17831, 62, 14421, 62, 16663, 3419, 4064, 25882, 62, 41359, 62, 4221, 15675, 50, 60, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 4808, 12501, 1098, 62, 30191, 39434, 944, 11, 16326, 25, 1222, 58, 385, 1096, 12962, 4613, 38692, 27, 84, 23, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 1005, 796, 38692, 3712, 4480, 62, 42404, 7, 83, 482, 641, 13, 11925, 3419, 1635, 362, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 329, 11241, 287, 16326, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 11241, 62, 33661, 796, 2116, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 12501, 12342, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 1136, 7, 30001, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 403, 37150, 62, 273, 62, 17772, 7, 15886, 1222, 944, 13, 20887, 62, 83, 482, 641, 62, 12501, 12342, 58, 30001, 36563, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1005, 13, 2302, 437, 7, 30001, 62, 33661, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1005, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 4808, 268, 8189, 62, 35947, 62, 30191, 39434, 944, 11, 2420, 25, 1222, 2536, 8, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 770, 318, 262, 4755, 286, 262, 21004, 9156, 26, 262, 584, 5499, 287, 994, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 655, 787, 1243, 8253, 47226, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 40364, 796, 2116, 13557, 1136, 62, 28781, 62, 260, 25636, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 1005, 796, 43030, 0, 58, 11208, 198, 220, 220, 220, 220, 220, 220, 220, 329, 2603, 287, 40364, 13, 19796, 62, 2676, 7, 5239, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 3704, 796, 2603, 13, 403, 37150, 22446, 292, 62, 2536, 22446, 292, 62, 33661, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1005, 13, 14689, 46491, 30001, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2555, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1005, 13, 2302, 437, 39434, 26327, 62, 24874, 62, 268, 8189, 7, 12239, 11, 1222, 944, 13, 12685, 12342, 18125, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1005, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 4808, 268, 8189, 62, 30191, 39434, 944, 11, 2420, 25, 1222, 2536, 11, 3142, 62, 20887, 25, 1222, 26257, 7248, 27, 5, 2536, 43734, 4613, 357, 53, 721, 27, 385, 1096, 22330, 514, 1096, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 2041, 62, 260, 25636, 796, 2116, 13557, 1136, 62, 28781, 62, 20887, 62, 260, 25636, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 40364, 796, 2116, 13557, 1136, 62, 28781, 62, 260, 25636, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 1005, 796, 43030, 0, 58, 11208, 628, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 923, 796, 657, 26, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 938, 62, 12239, 62, 30001, 62, 11925, 796, 657, 26, 198, 220, 220, 220, 220, 220, 220, 220, 9052, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 1306, 62, 20887, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 923, 62, 19796, 796, 923, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 9052, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 9938, 262, 1306, 3142, 2041, 11241, 11, 611, 597, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1306, 62, 20887, 796, 2041, 62, 260, 25636, 13, 19796, 62, 6738, 62, 1930, 7, 5239, 11, 923, 62, 19796, 737, 403, 37150, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2872, 1306, 62, 20887, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2773, 7, 76, 8, 5218, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 3142, 62, 20887, 13, 3642, 1299, 39434, 5239, 58, 76, 13, 9688, 3419, 492, 76, 13, 437, 3419, 12962, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2270, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 923, 62, 19796, 796, 285, 13, 9688, 3419, 1343, 352, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 6045, 5218, 2270, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 886, 796, 1306, 62, 20887, 13, 8899, 62, 273, 7, 5239, 13, 11925, 22784, 930, 76, 91, 285, 13, 9688, 35430, 628, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 16805, 11, 994, 356, 467, 11, 8996, 428, 9156, 284, 4808, 268, 8189, 62, 35947, 62, 30191, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 329, 2603, 287, 40364, 13, 19796, 62, 2676, 39434, 5239, 58, 9688, 492, 437, 12962, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 3704, 796, 2603, 13, 403, 37150, 22446, 292, 62, 2536, 22446, 292, 62, 33661, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 938, 62, 12239, 62, 30001, 62, 11925, 796, 352, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1005, 13, 14689, 46491, 30001, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2555, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 16326, 796, 18022, 62, 24874, 62, 268, 8189, 7, 12239, 11, 1222, 944, 13, 12685, 12342, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 938, 62, 12239, 62, 30001, 62, 11925, 796, 16326, 13, 11925, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1005, 13, 2302, 437, 39434, 83, 482, 641, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2872, 1306, 62, 20887, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 843, 994, 356, 4574, 262, 2041, 11241, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2773, 7, 76, 8, 5218, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 3704, 796, 285, 13, 292, 62, 2536, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 11241, 796, 2116, 13, 20887, 62, 83, 482, 641, 62, 12685, 12342, 58, 12239, 11208, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1005, 13, 14689, 7, 30001, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 923, 796, 285, 13, 437, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 938, 62, 12239, 62, 30001, 62, 11925, 796, 657, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 6045, 5218, 2270, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 3373, 938, 62, 12239, 62, 30001, 62, 11925, 318, 703, 867, 16326, 1625, 422, 262, 938, 40364, 6626, 13, 770, 318, 973, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 329, 13213, 21354, 16326, 11, 1201, 345, 460, 470, 20121, 1973, 357, 31284, 8, 40364, 30778, 198, 220, 220, 220, 220, 220, 220, 220, 357, 1186, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 4808, 24988, 589, 62, 12957, 62, 12239, 62, 30001, 62, 11925, 7, 198, 220, 220, 220, 220, 220, 220, 220, 1222, 944, 11, 198, 220, 220, 220, 220, 220, 220, 220, 16326, 25, 38692, 27, 385, 1096, 22330, 198, 220, 220, 220, 220, 220, 220, 220, 4517, 938, 62, 12239, 62, 30001, 62, 11925, 25, 514, 1096, 11, 198, 220, 220, 220, 1267, 4613, 357, 53, 721, 27, 385, 1096, 22330, 514, 1096, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 8989, 11, 262, 7064, 810, 674, 40364, 30778, 460, 307, 21354, 13, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 1114, 262, 4959, 286, 13213, 21354, 16326, 11, 21354, 40364, 26021, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 318, 691, 257, 1917, 611, 257, 6626, 326, 373, 1944, 27934, 11, 1201, 428, 460, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 1085, 284, 35981, 286, 16326, 4306, 1807, 284, 307, 8245, 13, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 537, 3064, 74, 62, 8692, 1838, 674, 1204, 1327, 416, 1390, 262, 3467, 82, 9, 58, 59, 81, 59, 77, 48688, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 3912, 13, 770, 460, 304, 13, 70, 13, 2728, 37082, 77, 1, 1343, 366, 366, 284, 1716, 37082, 77, 3467, 77, 1911, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 3423, 318, 257, 2068, 290, 11841, 4259, 25, 198, 220, 220, 220, 220, 220, 220, 220, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 11241, 62, 271, 62, 439, 62, 13200, 796, 930, 30001, 91, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2116, 13, 12501, 12342, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 1136, 7, 30001, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 8899, 7, 91, 30001, 62, 33661, 91, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 11241, 62, 33661, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 2676, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 18218, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 439, 7, 91, 5, 65, 91, 685, 65, 6, 46083, 275, 6, 59, 77, 3256, 275, 6, 59, 83, 6, 4083, 3642, 1299, 39434, 65, 4008, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 32092, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 403, 37150, 62, 273, 7, 9562, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 18083, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 938, 62, 12239, 62, 30001, 62, 11925, 1875, 657, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 11405, 11241, 62, 271, 62, 439, 62, 13200, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 12962, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 981, 357, 12957, 62, 12239, 62, 30001, 62, 11925, 1279, 16326, 13, 11925, 28955, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 11405, 11241, 62, 271, 62, 439, 62, 13200, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 532, 352, 12962, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 938, 62, 12239, 62, 30001, 62, 11925, 15853, 352, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 14257, 62, 30493, 0, 7, 12957, 62, 12239, 62, 30001, 62, 11925, 19841, 16326, 13, 11925, 35430, 628, 220, 220, 220, 220, 220, 220, 220, 357, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 4808, 268, 8189, 62, 403, 31284, 62, 30191, 7, 198, 220, 220, 220, 220, 220, 220, 220, 1222, 944, 11, 198, 220, 220, 220, 220, 220, 220, 220, 2420, 25, 1222, 2536, 11, 198, 220, 220, 220, 220, 220, 220, 220, 3142, 62, 20887, 25, 1222, 26257, 7248, 27, 5, 2536, 22330, 198, 220, 220, 220, 1267, 4613, 357, 53, 721, 27, 385, 1096, 22330, 21059, 7248, 27, 53, 721, 27, 385, 1096, 4211, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 357, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 2116, 13557, 268, 8189, 62, 30191, 7, 5239, 11, 3142, 62, 20887, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 611, 938, 62, 12239, 62, 30001, 62, 11925, 6624, 657, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 1002, 938, 62, 12239, 62, 30001, 62, 11925, 318, 6632, 11, 262, 938, 11241, 373, 257, 2041, 11241, 290, 356, 423, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 645, 21354, 9881, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1441, 357, 83, 482, 641, 11, 21059, 7248, 3712, 3605, 35430, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 357, 21973, 16326, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2116, 13557, 24988, 589, 62, 12957, 62, 12239, 62, 30001, 62, 11925, 7, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 628, 220, 220, 220, 220, 220, 220, 220, 1309, 21354, 62, 33661, 796, 2116, 13557, 12501, 1098, 62, 30191, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 492, 36563, 198, 220, 220, 220, 220, 220, 220, 220, 16326, 13, 2213, 19524, 378, 7, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 628, 220, 220, 220, 220, 220, 220, 220, 3373, 16926, 46, 25, 356, 815, 1949, 7069, 284, 1064, 3224, 8245, 16326, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 770, 561, 4646, 262, 2033, 286, 1005, 4233, 1710, 618, 13213, 1224, 45240, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 33973, 284, 262, 9156, 287, 281, 4697, 2196, 286, 428, 2393, 628, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 1224, 45240, 796, 21059, 7248, 3712, 3605, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 611, 21354, 62, 33661, 13, 271, 62, 28920, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1441, 357, 83, 482, 641, 11, 1224, 45240, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 3373, 770, 318, 262, 2562, 1643, 13, 2329, 1064, 477, 2060, 16326, 326, 923, 351, 21354, 62, 33661, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 357, 8201, 16326, 326, 3446, 2872, 21354, 62, 33661, 8, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 8621, 283, 803, 428, 422, 262, 9052, 2174, 5419, 351, 2854, 287, 257, 2219, 1339, 13, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 966, 796, 2116, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 82, 9741, 62, 30001, 62, 33661, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 3911, 653, 62, 4122, 7, 91, 87, 91, 2124, 13, 292, 62, 48369, 3419, 1279, 21354, 62, 33661, 13, 292, 62, 48369, 35430, 198, 220, 220, 220, 220, 220, 220, 220, 981, 966, 1279, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 13, 11925, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 11405, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 301, 5889, 62, 4480, 39434, 403, 31284, 62, 33661, 8, 198, 220, 220, 220, 220, 220, 220, 220, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1224, 45240, 13, 28463, 7, 35138, 0, 58, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2116, 13, 12685, 12342, 58, 944, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 292, 62, 48369, 3419, 4357, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2361, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 966, 15853, 352, 26, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 3373, 2735, 4174, 772, 517, 33908, 2700, 13, 1629, 790, 357, 847, 8, 1744, 2292, 329, 262, 965, 2860, 1359, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 11241, 11, 1673, 36686, 378, 3224, 9881, 422, 326, 11241, 357, 361, 597, 8, 284, 21354, 62, 33661, 11, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 290, 1005, 4233, 786, 262, 2187, 1517, 290, 766, 644, 356, 651, 13, 198, 220, 220, 220, 220, 220, 220, 220, 329, 1312, 287, 352, 492, 403, 31284, 62, 33661, 13, 11925, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 21231, 796, 1222, 403, 31284, 62, 33661, 58, 492, 72, 11208, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 35488, 796, 1222, 403, 31284, 62, 33661, 58, 72, 492, 11208, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 966, 796, 2116, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 82, 9741, 62, 30001, 62, 33661, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 3911, 653, 62, 4122, 7, 91, 87, 91, 2124, 13, 292, 62, 48369, 3419, 1279, 35488, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 16926, 46, 25, 2448, 69, 6436, 5612, 611, 35488, 4940, 351, 366, 366, 30, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 981, 966, 1279, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 13, 11925, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 11405, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 301, 5889, 62, 4480, 7, 37333, 844, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 5885, 796, 685, 40290, 11, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 292, 62, 48369, 3419, 4083, 1102, 9246, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 30240, 796, 2872, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 39434, 79, 43691, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 3461, 453, 11, 428, 318, 18022, 62, 24874, 62, 268, 8189, 39434, 79, 43691, 11, 1222, 944, 13, 12685, 12342, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 887, 356, 1244, 423, 5495, 257, 40364, 6626, 543, 561, 2948, 4017, 3212, 13, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 357, 31722, 1744, 287, 262, 4931, 286, 21354, 40364, 30778, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 1406, 10385, 284, 41002, 12, 23, 290, 466, 40364, 26021, 13, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 412, 13, 70, 13, 351, 537, 3064, 74, 62, 8692, 366, 220, 220, 2474, 3011, 6626, 284, 366, 366, 1343, 366, 5145, 1600, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 475, 18022, 62, 24874, 62, 268, 8189, 7203, 220, 220, 2474, 8, 14512, 18022, 62, 24874, 62, 268, 8189, 7203, 366, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 6762, 7, 82, 8, 5218, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 82, 828, 628, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 5429, 1146, 11, 1771, 393, 407, 428, 3211, 318, 3376, 8338, 319, 1771, 612, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 561, 307, 257, 40364, 6626, 878, 262, 41002, 12, 23, 40122, 341, 966, 13, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 18578, 21404, 1576, 326, 645, 530, 481, 1683, 4003, 357, 8499, 477, 11, 661, 1422, 470, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 4003, 477, 262, 1263, 10421, 287, 262, 2180, 21354, 11241, 7822, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 41512, 28264, 8, 5218, 18022, 62, 24874, 62, 268, 8189, 39434, 79, 43691, 11, 1222, 944, 13, 12685, 12342, 828, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 13742, 588, 262, 1708, 318, 19827, 475, 11491, 25, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 41512, 7, 68, 8, 5218, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 13271, 8635, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 220, 220, 220, 220, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 62, 403, 26752, 39434, 79, 43691, 58, 492, 68, 13, 12102, 62, 929, 62, 1462, 3419, 12962, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 1782, 828, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 18083, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 33756, 796, 38692, 3712, 3605, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 33756, 62, 11925, 796, 657, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 329, 11241, 287, 30240, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 33756, 13, 14689, 7, 30001, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 33756, 62, 11925, 15853, 2116, 13, 12501, 12342, 58, 5, 30001, 4083, 11925, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 33756, 62, 11925, 18189, 21354, 62, 33661, 13, 11925, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2270, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1224, 45240, 13, 28463, 7, 41068, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 966, 15853, 352, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 3373, 770, 318, 635, 407, 15836, 13, 2893, 356, 4143, 7048, 326, 40364, 30778, 389, 8245, 11, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 12716, 11, 484, 389, 407, 13, 1320, 318, 11, 611, 4375, 9881, 547, 284, 787, 257, 6626, 1656, 287, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 21354, 62, 33661, 11, 428, 714, 787, 16326, 1744, 543, 674, 9156, 561, 4306, 892, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 561, 307, 23791, 13, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 1114, 1672, 11, 351, 308, 457, 17, 11, 262, 779, 286, 3467, 82, 33747, 12248, 59, 50, 8, 1724, 326, 37082, 77, 59, 77, 1, 714, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 1205, 257, 6626, 11, 304, 13, 70, 13, 37082, 77, 59, 77, 15, 1, 30778, 656, 37082, 77, 1, 10, 1, 59, 77, 1, 10, 1, 15, 1600, 1642, 37082, 77, 1, 257, 1744, 11241, 13, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 3423, 318, 257, 2068, 290, 11841, 4259, 25, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 770, 2125, 470, 826, 611, 356, 1683, 4781, 3467, 82, 33747, 12248, 59, 50, 8, 198, 220, 220, 220, 220, 220, 220, 220, 611, 21354, 62, 33661, 13, 11925, 3419, 1875, 352, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 938, 62, 12501, 9043, 796, 275, 2536, 3712, 12501, 1098, 62, 12957, 62, 40477, 23, 7, 403, 31284, 62, 33661, 13, 292, 62, 48369, 35430, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 21354, 62, 33661, 13, 11925, 3419, 532, 938, 62, 12501, 9043, 13, 16, 1875, 657, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 11405, 938, 62, 12501, 9043, 13, 15, 13, 8899, 62, 273, 7, 9562, 11, 930, 66, 91, 269, 13, 271, 62, 1929, 2737, 10223, 28955, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 302, 12685, 9043, 796, 18022, 62, 24874, 62, 268, 8189, 7, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1222, 403, 31284, 62, 33661, 58, 492, 403, 31284, 62, 33661, 13, 11925, 3419, 532, 938, 62, 12501, 9043, 13, 16, 4357, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1222, 944, 13, 12685, 12342, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 5619, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 302, 12685, 9043, 13, 2302, 437, 7, 26327, 62, 24874, 62, 268, 8189, 7, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1222, 403, 31284, 62, 33661, 58, 403, 31284, 62, 33661, 13, 11925, 3419, 532, 938, 62, 12501, 9043, 13, 16, 492, 4357, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1222, 944, 13, 12685, 12342, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 29226, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1224, 45240, 13, 28463, 7, 1361, 40976, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 357, 83, 482, 641, 11, 1224, 45240, 8, 198, 220, 220, 220, 1782, 198, 92, 198, 198, 2, 58, 9078, 24396, 82, 60, 198, 23928, 7231, 33, 11401, 1391, 198, 220, 220, 220, 1303, 58, 3605, 60, 198, 220, 220, 220, 24714, 649, 7, 198, 220, 220, 220, 220, 220, 220, 220, 2207, 12342, 25, 21059, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 22330, 198, 220, 220, 220, 220, 220, 220, 220, 2041, 62, 83, 482, 641, 62, 12685, 12342, 25, 21059, 13912, 27, 10100, 11, 514, 1096, 22330, 198, 220, 220, 220, 220, 220, 220, 220, 3912, 25, 1222, 2536, 11, 198, 220, 220, 220, 1267, 4613, 9485, 23004, 27, 24704, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 40364, 796, 797, 25636, 3712, 3605, 7, 33279, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 8899, 62, 8056, 7, 91, 68, 91, 9485, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 11395, 12331, 11, 4808, 33994, 68, 13, 1462, 62, 8841, 3419, 4008, 30, 26, 628, 220, 220, 220, 220, 220, 220, 220, 1309, 2041, 62, 260, 25636, 796, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4808, 42632, 796, 2041, 62, 83, 482, 641, 62, 12685, 12342, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 13083, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 8899, 7, 91, 82, 91, 14996, 62, 260, 25636, 3712, 41915, 7, 82, 4008, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 33327, 3712, 27, 53, 721, 27, 62, 4211, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 797, 25636, 3712, 3605, 39434, 62, 42632, 13, 22179, 7203, 91, 48774, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 8899, 62, 8056, 7, 91, 68, 91, 9485, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 11395, 12331, 11, 4808, 33994, 68, 13, 1462, 62, 8841, 3419, 4008, 30, 198, 220, 220, 220, 220, 220, 220, 220, 18083, 628, 220, 220, 220, 220, 220, 220, 220, 1309, 875, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 796, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2207, 12342, 13, 2676, 22446, 8899, 7, 91, 7, 74, 11, 410, 14726, 20789, 85, 11, 479, 13, 21018, 28955, 737, 33327, 9783, 628, 220, 220, 220, 220, 220, 220, 220, 6818, 0, 7, 12685, 12342, 13, 11925, 3419, 6624, 875, 12342, 13, 11925, 35430, 628, 220, 220, 220, 220, 220, 220, 220, 1309, 2041, 62, 83, 482, 641, 62, 12501, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 796, 2041, 62, 83, 482, 641, 62, 12685, 12342, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 2676, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 8899, 7, 91, 7, 74, 11, 410, 14726, 20789, 85, 11, 479, 13, 292, 62, 33661, 22446, 1462, 62, 35138, 3419, 4008, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 33327, 9783, 628, 220, 220, 220, 220, 220, 220, 220, 3373, 30698, 780, 314, 836, 470, 760, 703, 284, 1560, 17103, 314, 1101, 407, 1016, 284, 1487, 262, 3975, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 23243, 62, 30001, 62, 33661, 25, 38692, 27, 53, 721, 27, 84, 23, 4211, 796, 2207, 12342, 13, 13083, 22446, 565, 12004, 22446, 33327, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 23243, 62, 30001, 62, 33661, 13, 30619, 9783, 628, 220, 220, 220, 220, 220, 220, 220, 6762, 7, 14055, 33, 11401, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2207, 12342, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2041, 62, 83, 482, 641, 62, 12685, 12342, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 875, 12342, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2041, 62, 83, 482, 641, 62, 12501, 12342, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 40364, 62, 83, 7278, 25, 357, 15, 492, 22921, 62, 41359, 62, 4221, 15675, 50, 737, 8899, 7, 91, 62, 91, 40364, 13, 21018, 3419, 737, 33327, 22784, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2041, 62, 260, 25636, 62, 83, 7278, 25, 357, 15, 492, 22921, 62, 41359, 62, 4221, 15675, 50, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 8899, 7, 91, 62, 91, 2041, 62, 260, 25636, 13, 21018, 28955, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 33327, 22784, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 23243, 62, 30001, 62, 33661, 11, 198, 220, 220, 220, 220, 220, 220, 220, 32092, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 3373, 36658, 18604, 198, 220, 220, 220, 3373, 14711, 7656, 198, 220, 220, 220, 3373, 36658, 18604, 628, 220, 220, 220, 24714, 37773, 62, 35947, 39434, 944, 11, 12972, 25, 11361, 11, 2420, 25, 1222, 2536, 8, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 5239, 4008, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 37773, 39434, 944, 11, 12972, 25, 11361, 11, 2420, 25, 1222, 2536, 11, 3142, 62, 20887, 25, 21059, 7248, 27, 5, 2536, 43734, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 268, 8189, 62, 30191, 7, 5239, 11, 1222, 40845, 62, 20887, 737, 15, 8, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 4808, 268, 8189, 62, 33661, 39434, 944, 11, 12972, 25, 11361, 11, 9881, 25, 1222, 58, 84, 23, 12962, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2872, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 7, 33661, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 6762, 7, 5239, 8, 5218, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 5239, 828, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 41512, 7, 68, 8, 5218, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 2420, 796, 21596, 1391, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 62, 403, 26752, 39434, 33661, 58, 492, 68, 13, 12102, 62, 929, 62, 1462, 3419, 12962, 18083, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 357, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 2116, 13557, 268, 8189, 62, 30191, 7, 5239, 11, 1222, 26257, 7248, 3712, 3605, 35430, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 357, 21973, 16326, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2116, 13557, 24988, 589, 62, 12957, 62, 12239, 62, 30001, 62, 11925, 7, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 5145, 83, 482, 641, 13, 271, 62, 28920, 3419, 11405, 938, 62, 12239, 62, 30001, 62, 11925, 1875, 657, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 406, 404, 572, 262, 16326, 422, 262, 938, 3704, 290, 1057, 347, 11401, 319, 262, 5637, 9881, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 32499, 5183, 21404, 11, 475, 428, 743, 407, 307, 3376, 611, 356, 1549, 423, 550, 257, 40364, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 6626, 1022, 262, 4938, 41002, 12, 23, 290, 262, 12515, 9881, 11, 543, 318, 1521, 428, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 2446, 318, 2839, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 21354, 62, 33661, 796, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2116, 13557, 12501, 1098, 62, 30191, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 492, 36563, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 21354, 62, 33661, 13, 2302, 437, 62, 6738, 62, 48369, 39434, 33661, 58, 68, 13, 12102, 62, 929, 62, 1462, 3419, 492, 36563, 628, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 16326, 13, 2213, 19524, 378, 7, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 16326, 13, 2302, 437, 7, 26327, 62, 24874, 62, 268, 8189, 39434, 403, 31284, 62, 33661, 11, 1222, 944, 13, 12685, 12342, 18125, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 16326, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 32092, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 37773, 62, 4480, 62, 403, 31284, 7, 198, 220, 220, 220, 220, 220, 220, 220, 1222, 944, 11, 198, 220, 220, 220, 220, 220, 220, 220, 12972, 25, 11361, 11, 198, 220, 220, 220, 220, 220, 220, 220, 2420, 25, 1222, 2536, 11, 198, 220, 220, 220, 220, 220, 220, 220, 3142, 62, 20887, 25, 21059, 7248, 27, 5, 2536, 22330, 198, 220, 220, 220, 1267, 4613, 9485, 27, 20519, 51, 29291, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 357, 83, 482, 641, 11, 1224, 45240, 8, 796, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 268, 8189, 62, 403, 31284, 62, 30191, 7, 5239, 11, 1222, 40845, 62, 20887, 18125, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 12972, 62, 785, 37069, 507, 796, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 9485, 8053, 3712, 3605, 7, 9078, 11, 1224, 45240, 13, 2676, 22446, 8899, 7, 91, 41068, 91, 9485, 8053, 3712, 3605, 7, 9078, 11, 1222, 41068, 58, 492, 60, 4008, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 357, 83, 482, 641, 11, 12972, 62, 785, 37069, 507, 737, 20424, 62, 9078, 7, 9078, 8, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 37773, 62, 29762, 62, 30001, 39434, 944, 11, 3704, 25, 1222, 58, 84, 23, 12962, 4613, 9485, 23004, 27, 385, 1096, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 737, 22163, 798, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1441, 6762, 7, 30001, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 6762, 7, 12239, 62, 2536, 8, 796, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 7, 12239, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 20887, 62, 83, 482, 641, 62, 12685, 12342, 13, 1136, 7, 12239, 62, 2536, 737, 22163, 798, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1441, 6762, 7, 30001, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 41512, 7, 20519, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 9218, 12331, 11, 4808, 33994, 12239, 13, 1462, 62, 11990, 3419, 4008, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 37773, 62, 29762, 62, 12239, 39434, 944, 11, 3704, 25, 1222, 58, 84, 23, 12962, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1441, 43030, 0, 58, 9, 30001, 11208, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 18022, 62, 24874, 62, 268, 8189, 7, 12239, 11, 1222, 944, 13, 12685, 12342, 8, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 3373, 36658, 18604, 198, 220, 220, 220, 3373, 4280, 7656, 198, 220, 220, 220, 3373, 36658, 18604, 628, 220, 220, 220, 24714, 36899, 62, 33661, 39434, 944, 11, 12972, 25, 11361, 11, 16326, 25, 38692, 27, 385, 1096, 43734, 4613, 9485, 27, 20519, 45992, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 9881, 796, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 12501, 1098, 62, 30191, 39434, 83, 482, 641, 18125, 198, 220, 220, 220, 220, 220, 220, 220, 9485, 45992, 3712, 3605, 7, 9078, 11, 1222, 33661, 737, 20424, 3419, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 36899, 62, 29762, 62, 30001, 62, 33661, 39434, 944, 11, 12972, 25, 11361, 11, 11241, 25, 514, 1096, 8, 4613, 9485, 23004, 27, 20519, 27, 20519, 45992, 4211, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 2773, 7, 33661, 8, 796, 2116, 13, 12501, 12342, 13, 1136, 39434, 30001, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1441, 6762, 7, 20519, 45992, 3712, 3605, 7, 9078, 11, 9881, 737, 20424, 35430, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 2773, 7, 33661, 8, 796, 2116, 13, 20887, 62, 83, 482, 641, 62, 12501, 12342, 13, 1136, 39434, 30001, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1441, 6762, 7, 20519, 45992, 3712, 3605, 7, 9078, 11, 9881, 737, 20424, 35430, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 41512, 7, 20519, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 9218, 12331, 11, 4808, 33994, 30001, 13, 1462, 62, 8841, 3419, 4008, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 3373, 36658, 18604, 198, 220, 220, 220, 3373, 46253, 198, 220, 220, 220, 3373, 36658, 18604, 628, 220, 220, 220, 24714, 11241, 62, 26327, 62, 27160, 39434, 944, 11, 12972, 25, 11361, 8, 4613, 38692, 27, 20519, 27, 20519, 45992, 4211, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 2676, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 8899, 7, 91, 87, 91, 9485, 45992, 3712, 3605, 7, 9078, 11, 2124, 737, 20424, 28955, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 33327, 3419, 198, 220, 220, 220, 1782, 198, 92, 198, 198, 2, 58, 9078, 21412, 60, 198, 22184, 4808, 83, 1134, 30001, 28264, 9078, 25, 11361, 11, 285, 25, 1222, 20519, 26796, 8, 4613, 9485, 23004, 27, 3419, 29, 1391, 198, 220, 220, 220, 285, 13, 2860, 62, 4871, 3712, 27, 14055, 33, 11401, 29, 3419, 30, 26, 198, 220, 220, 220, 6762, 7, 28955, 198, 92, 198, 198, 2, 58, 37581, 7, 9288, 15437, 198, 4666, 5254, 1391, 198, 220, 220, 220, 779, 17000, 66, 62, 17831, 3712, 37, 87, 26257, 13912, 355, 21059, 13912, 26, 628, 220, 220, 220, 779, 27021, 3712, 26327, 62, 24874, 62, 35312, 26, 628, 220, 220, 220, 1303, 58, 9288, 60, 198, 220, 220, 220, 24714, 845, 62, 36439, 62, 9288, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 9803, 796, 21059, 13912, 3712, 12286, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 9803, 13, 28463, 7, 65, 1, 397, 1911, 1462, 62, 35138, 22784, 352, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 9803, 13, 28463, 7, 65, 1, 10210, 1911, 1462, 62, 35138, 22784, 362, 1776, 628, 220, 220, 220, 220, 220, 220, 220, 1309, 581, 796, 18022, 62, 24874, 62, 35312, 7, 65, 1, 397, 10210, 1600, 1222, 81, 2283, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 6818, 62, 27363, 0, 7, 411, 11, 43030, 0, 58, 65, 1, 397, 1600, 275, 1, 10210, 8973, 1776, 198, 220, 220, 220, 1782, 198, 92, 198] \ No newline at end of file diff --git a/test/Microsoft.ML.Tokenizers.Tests/Data/tokens_p50k_base.json b/test/Microsoft.ML.Tokenizers.Tests/Data/tokens_p50k_base.json new file mode 100644 index 0000000000..54efe06336 --- /dev/null +++ b/test/Microsoft.ML.Tokenizers.Tests/Data/tokens_p50k_base.json @@ -0,0 +1 @@ +[1003, 770, 2198, 318, 649, 290, 2331, 46542, 357, 39363, 351, 9485, 46, 18, 10375, 8, 198, 2, 0, 58, 12154, 7, 565, 41214, 3712, 2865, 808, 62, 67, 567, 69, 62, 5420, 15437, 198, 198, 1904, 14367, 3712, 4033, 26448, 3712, 26257, 7248, 26, 198, 1904, 14367, 3712, 16663, 26, 198, 198, 1904, 14996, 62, 260, 25636, 3712, 3041, 25636, 26, 198, 1904, 279, 8226, 18, 3712, 1069, 11755, 26, 198, 1904, 279, 8226, 18, 3712, 79, 2411, 2507, 3712, 9, 26, 198, 1904, 279, 8226, 18, 3712, 19199, 3712, 90, 20519, 45992, 11, 9485, 8053, 11, 9485, 51, 29291, 19629, 198, 1904, 279, 8226, 18, 3712, 20519, 23004, 26, 198, 1904, 17000, 66, 62, 17831, 3712, 37, 87, 26257, 13912, 355, 21059, 13912, 26, 198, 198, 22184, 4808, 26327, 62, 24874, 62, 647, 469, 27, 51, 33994, 198, 50258, 3704, 25, 1222, 58, 84, 23, 4357, 198, 50258, 9803, 25, 1222, 26257, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 22330, 198, 50258, 277, 25, 4114, 37481, 7, 19282, 3712, 2840, 3712, 17257, 27, 385, 1096, 43734, 4613, 309, 11, 198, 8, 4613, 38692, 27, 51, 29, 1391, 198, 50258, 3373, 770, 318, 257, 15879, 286, 357, 9688, 11, 4279, 737, 198, 50258, 3373, 383, 4279, 318, 286, 262, 18022, 5166, 3599, 379, 2292, 923, 13, 198, 50258, 3373, 383, 4279, 286, 262, 938, 2378, 287, 262, 15879, 318, 407, 257, 4938, 1988, 13, 198, 50258, 1309, 4517, 3354, 25, 38692, 27, 7, 385, 1096, 11, 514, 1096, 8, 29, 796, 357, 15, 492, 12239, 13, 11925, 3419, 1343, 352, 737, 8899, 7, 91, 72, 91, 357, 72, 11, 514, 1096, 3712, 22921, 29720, 33327, 9783, 628, 50258, 1309, 651, 62, 43027, 796, 1391, 198, 50262, 1303, 58, 45145, 7, 33770, 15437, 198, 50262, 930, 42632, 25, 1222, 53, 721, 27, 7, 385, 1096, 11, 514, 1096, 8, 22330, 923, 62, 312, 87, 25, 514, 1096, 11, 14267, 25, 514, 1096, 91, 1391, 198, 50266, 611, 357, 9688, 62, 312, 87, 1343, 14267, 1343, 362, 8, 1279, 3354, 13, 11925, 3419, 1391, 198, 50270, 9803, 198, 50274, 764, 1136, 39434, 12239, 58, 42632, 58, 9688, 62, 312, 87, 4083, 15, 492, 42632, 58, 9688, 62, 312, 87, 1343, 14267, 1343, 362, 4083, 15, 12962, 198, 50274, 764, 22163, 798, 3419, 198, 50266, 1782, 2073, 1391, 198, 50270, 6045, 198, 50266, 1782, 198, 50262, 1782, 198, 50258, 18083, 628, 50258, 3373, 775, 804, 510, 262, 9803, 1752, 287, 262, 3726, 290, 11629, 9404, 4296, 198, 50258, 3373, 606, 1141, 1123, 20121, 11, 543, 12850, 262, 1271, 286, 4279, 804, 4739, 13, 198, 50258, 329, 1312, 287, 657, 492, 42632, 13, 11925, 3419, 532, 362, 1391, 198, 50262, 2872, 651, 62, 43027, 39434, 42632, 11, 1312, 11, 657, 8, 1391, 198, 50266, 2773, 7, 43027, 8, 5218, 1391, 198, 50270, 3373, 514, 1096, 3712, 22921, 318, 257, 1908, 20538, 1988, 290, 2314, 307, 257, 4938, 4279, 198, 50270, 14257, 62, 30493, 0, 7, 43027, 14512, 514, 1096, 3712, 22921, 1776, 198, 50270, 3354, 58, 72, 4083, 16, 796, 4279, 26, 198, 50266, 1782, 198, 50266, 6045, 5218, 1391, 198, 50270, 2555, 26, 198, 50266, 1782, 198, 50262, 18083, 198, 50258, 1782, 628, 50258, 3373, 1002, 345, 423, 299, 3354, 290, 285, 4017, 3212, 11, 428, 857, 440, 7, 10295, 8, 670, 13, 198, 50258, 3373, 775, 714, 466, 1223, 351, 257, 24575, 290, 466, 440, 7, 76, 2604, 299, 8, 670, 13, 198, 50258, 3373, 632, 318, 1593, 284, 2074, 326, 299, 318, 1690, 1402, 38155, 3064, 828, 290, 355, 884, 198, 50258, 3373, 262, 12940, 12, 17946, 1483, 4034, 31405, 262, 8385, 9383, 13357, 21838, 1460, 198, 50258, 3373, 286, 262, 4600, 42632, 63, 15879, 1366, 4645, 2029, 13, 628, 50258, 3373, 5740, 326, 356, 12234, 9881, 11, 407, 11241, 14729, 13, 1081, 890, 355, 356, 4512, 347, 11401, 262, 835, 356, 198, 50258, 3373, 3058, 466, 11, 428, 318, 7548, 13, 1052, 2562, 835, 284, 2270, 428, 561, 307, 284, 875, 43846, 198, 50258, 3373, 20121, 8475, 422, 11241, 6376, 393, 284, 2948, 2176, 11241, 4017, 3212, 13, 198, 50258, 9052, 1391, 198, 50262, 611, 3354, 13, 11925, 3419, 6624, 352, 1391, 198, 50266, 2270, 26, 198, 50262, 1782, 628, 50262, 3373, 514, 1096, 3712, 22921, 318, 257, 1908, 20538, 4279, 1988, 5086, 514, 284, 198, 50262, 3373, 1011, 262, 949, 517, 2952, 198, 50262, 1309, 4517, 949, 62, 43027, 25, 357, 385, 1096, 11, 514, 1096, 8, 796, 357, 385, 1096, 3712, 22921, 11, 657, 1776, 198, 50262, 329, 357, 72, 11, 1222, 28264, 11, 4279, 4008, 287, 3354, 58, 492, 42632, 13, 11925, 3419, 532, 352, 4083, 2676, 22446, 268, 6975, 378, 3419, 1391, 198, 50266, 611, 4279, 1279, 949, 62, 43027, 13, 15, 1391, 198, 50270, 949, 62, 43027, 796, 357, 43027, 11, 1312, 1776, 198, 50266, 1782, 198, 50262, 1782, 628, 50262, 611, 949, 62, 43027, 13, 15, 14512, 514, 1096, 3712, 22921, 1391, 198, 50266, 1309, 1312, 796, 949, 62, 43027, 13, 16, 26, 628, 50266, 3373, 24550, 25, 775, 389, 546, 284, 4781, 3354, 58, 72, 1343, 352, 4083, 775, 466, 407, 466, 340, 198, 50266, 3373, 1865, 780, 612, 389, 12940, 12, 17946, 1483, 4034, 284, 19698, 198, 50266, 3373, 3354, 58, 72, 60, 290, 3354, 58, 72, 12, 16, 60, 878, 10829, 11, 543, 714, 5636, 1077, 198, 50266, 3373, 262, 12940, 13, 6660, 11, 356, 4296, 262, 4279, 17952, 416, 31017, 625, 198, 50266, 3373, 3354, 58, 72, 1343, 352, 4357, 416, 39744, 4600, 1136, 62, 43027, 0, 63, 351, 4600, 48267, 796, 352, 44646, 198, 50266, 3354, 58, 72, 4083, 16, 796, 651, 62, 43027, 39434, 42632, 11, 1312, 11, 352, 737, 403, 37150, 62, 273, 7, 385, 1096, 3712, 22921, 1776, 198, 50266, 611, 1312, 1875, 657, 1391, 198, 50270, 3354, 58, 72, 532, 352, 4083, 16, 796, 651, 62, 43027, 39434, 42632, 11, 1312, 532, 352, 11, 352, 737, 403, 37150, 62, 273, 7, 385, 1096, 3712, 22921, 1776, 198, 50266, 1782, 628, 50266, 3354, 13, 28956, 7, 72, 1343, 352, 1776, 198, 50262, 1782, 2073, 1391, 198, 50266, 2270, 26, 198, 50262, 1782, 198, 50258, 1782, 198, 50258, 1309, 4517, 503, 25, 38692, 27, 51, 29, 796, 38692, 3712, 4480, 62, 42404, 7, 42632, 13, 11925, 3419, 532, 352, 1776, 198, 50258, 329, 1312, 287, 657, 492, 42632, 13, 11925, 3419, 532, 352, 1391, 198, 50262, 503, 13, 14689, 7, 69, 7, 42632, 58, 72, 4083, 15, 492, 42632, 58, 72, 1343, 352, 4083, 15, 18125, 198, 50258, 1782, 198, 50258, 503, 198, 92, 198, 198, 12984, 24714, 18022, 62, 24874, 62, 268, 8189, 7, 12239, 25, 1222, 58, 84, 23, 4357, 9803, 25, 1222, 26257, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 43734, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 50258, 611, 3704, 13, 11925, 3419, 6624, 352, 1391, 198, 50262, 1441, 43030, 0, 58, 81, 2283, 58, 12239, 60, 11208, 198, 50258, 1782, 198, 50258, 4808, 26327, 62, 24874, 62, 647, 469, 7, 12239, 11, 9803, 11, 930, 79, 91, 9803, 58, 5, 12239, 58, 79, 13, 9688, 492, 79, 13, 437, 11907, 8, 198, 92, 198, 198, 12984, 24714, 18022, 62, 24874, 62, 35312, 27, 6, 64, 33994, 12239, 25, 1222, 6, 64, 685, 84, 23, 4357, 9803, 25, 1222, 26257, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 43734, 4613, 38692, 27, 5, 6, 64, 685, 84, 23, 60, 29, 1391, 198, 50258, 611, 3704, 13, 11925, 3419, 6624, 352, 1391, 198, 50262, 1441, 43030, 0, 58, 12239, 11208, 198, 50258, 1782, 198, 50258, 4808, 26327, 62, 24874, 62, 647, 469, 7, 12239, 11, 9803, 11, 930, 79, 91, 1222, 12239, 58, 79, 13, 9688, 492, 79, 13, 437, 12962, 198, 92, 198, 198, 1003, 26386, 2854, 4710, 25, 198, 1003, 198, 1003, 797, 25636, 198, 1003, 29335, 198, 1003, 4042, 286, 262, 640, 318, 3377, 287, 40364, 13, 383, 16638, 835, 284, 2866, 428, 510, 318, 416, 1262, 1342, 14996, 198, 1003, 40364, 3033, 13, 1114, 4554, 11, 1262, 257, 40364, 21136, 12, 540, 416, 4600, 260, 25636, 63, 27021, 318, 513, 87, 5443, 621, 198, 1003, 262, 6678, 40364, 356, 779, 13, 198, 1003, 198, 1003, 2102, 11, 1813, 326, 356, 821, 1262, 257, 40364, 21136, 12, 540, 416, 4600, 260, 25636, 47671, 612, 2125, 470, 881, 3580, 198, 1003, 1022, 1262, 262, 4600, 260, 25636, 63, 27021, 290, 1262, 262, 4600, 69, 3883, 62, 260, 25636, 63, 27021, 13, 198, 1003, 198, 1003, 1318, 318, 281, 1593, 10375, 1022, 4704, 278, 11, 4600, 260, 25636, 63, 290, 4600, 69, 3883, 62, 260, 25636, 44646, 198, 1003, 1649, 1262, 4600, 69, 3883, 62, 260, 25636, 47671, 356, 2277, 4600, 260, 25636, 13, 19796, 62, 265, 44646, 632, 4962, 503, 326, 428, 5640, 22627, 319, 198, 1003, 617, 4517, 540, 12692, 2272, 2641, 286, 4600, 260, 25636, 44646, 770, 5543, 12847, 2854, 13, 1649, 1262, 8631, 198, 1003, 1468, 4600, 260, 25636, 47671, 356, 836, 470, 2277, 428, 11, 780, 4600, 19796, 62, 2676, 63, 468, 257, 1180, 2438, 3108, 13, 198, 1003, 19809, 25, 3740, 1378, 12567, 13, 785, 14, 11469, 12, 17204, 14, 260, 25636, 14, 2436, 672, 14, 9866, 14, 18973, 13775, 10725, 5222, 13, 9132, 198, 1003, 21836, 11, 262, 835, 356, 651, 1088, 428, 318, 351, 1719, 257, 357, 29471, 8, 4704, 1957, 17271, 286, 262, 40364, 329, 198, 1003, 1123, 4704, 13, 198, 1003, 198, 1003, 14122, 278, 198, 1003, 796, 2559, 198, 1003, 314, 3088, 1262, 4600, 2433, 261, 44646, 632, 2492, 470, 1107, 5443, 621, 1262, 11361, 14390, 290, 13011, 262, 402, 4146, 13, 198, 1003, 1406, 24829, 4600, 2433, 261, 63, 0, 3914, 4704, 954, 3503, 307, 287, 1630, 286, 674, 11361, 2985, 13, 198, 1003, 198, 1003, 327, 8103, 198, 1003, 29335, 855, 198, 1003, 383, 4941, 11241, 5847, 468, 281, 300, 622, 12940, 625, 262, 7548, 286, 4600, 26327, 62, 24874, 62, 268, 8189, 44646, 198, 1003, 19486, 11, 356, 550, 530, 1165, 0, 9170, 340, 11, 356, 547, 691, 29627, 5443, 621, 11361, 13, 198, 1003, 314, 973, 281, 33212, 25392, 284, 1805, 262, 12940, 13, 770, 1422, 470, 1283, 284, 5938, 2060, 40945, 2854, 198, 1003, 33752, 11, 475, 340, 750, 2689, 5021, 12, 16663, 276, 2854, 13, 34097, 306, 11, 340, 3947, 284, 2689, 198, 1003, 5021, 12, 16663, 276, 2854, 772, 618, 314, 691, 550, 7183, 357, 11261, 3077, 314, 32621, 1223, 510, 29865, 198, 1003, 21836, 11, 314, 19169, 326, 356, 714, 651, 5755, 286, 262, 12940, 11, 611, 356, 2190, 262, 900, 286, 16326, 355, 257, 12940, 0, 198, 1003, 2312, 389, 3446, 262, 900, 393, 4017, 3212, 326, 389, 1884, 284, 307, 3024, 13, 843, 783, 356, 836, 470, 423, 284, 892, 198, 1003, 546, 11087, 4517, 1799, 11, 4088, 779, 11, 393, 45973, 13, 198, 1003, 198, 1003, 367, 2140, 198, 1003, 29335, 855, 198, 1003, 775, 779, 376, 87, 26257, 13912, 2427, 286, 262, 3210, 21059, 13912, 13, 770, 318, 3863, 588, 257, 642, 12, 940, 4, 1592, 30, 198, 1003, 383, 1459, 7822, 5645, 510, 1804, 257, 1256, 286, 49544, 286, 9881, 13, 554, 4583, 11, 428, 714, 307, 925, 198, 1003, 284, 307, 49544, 286, 734, 12, 28047, 2374, 286, 493, 82, 11, 543, 3073, 588, 340, 743, 635, 307, 257, 3155, 1411, 5443, 13, 198, 198, 1904, 14367, 3712, 22510, 3712, 15419, 28667, 52, 2414, 26, 198, 12984, 2878, 33482, 16818, 7390, 7, 15419, 28667, 52, 2414, 1776, 198, 198, 22184, 12234, 62, 14421, 62, 16663, 3419, 4613, 514, 1096, 1391, 198, 50258, 3373, 632, 338, 4577, 284, 779, 21596, 621, 284, 779, 37862, 13, 17103, 468, 428, 3621, 334, 2414, 4704, 4686, 3753, 198, 50258, 3373, 326, 2499, 1049, 329, 674, 779, 1339, 286, 14928, 31998, 287, 674, 7177, 13, 8989, 11, 198, 50258, 3373, 340, 338, 2839, 13, 2102, 11, 612, 389, 691, 523, 867, 2842, 345, 460, 12461, 257, 334, 2414, 11, 523, 655, 21595, 1133, 198, 50258, 3373, 3740, 1378, 12567, 13, 785, 14, 11469, 12, 17204, 14, 11469, 14, 37165, 14, 37601, 2670, 198, 50258, 1500, 4808, 25, 685, 84, 23, 26, 807, 60, 796, 685, 15, 26, 14367, 3712, 11883, 3712, 7857, 62, 1659, 3712, 27, 19282, 3712, 16663, 3712, 16818, 7390, 29, 3419, 11208, 198, 50258, 1500, 4808, 25, 685, 84, 23, 26, 807, 60, 796, 685, 15, 26, 14367, 3712, 11883, 3712, 7857, 62, 1659, 3712, 27, 49233, 16818, 7390, 29, 3419, 11208, 198, 50258, 1309, 2124, 796, 21596, 1391, 198, 50262, 14367, 3712, 11883, 3712, 7645, 76, 1133, 3712, 27, 19282, 3712, 16663, 3712, 16818, 7390, 11, 33482, 16818, 7390, 33994, 16663, 3712, 14421, 22446, 312, 3419, 737, 15, 198, 50258, 18083, 198, 50258, 334, 2414, 3712, 6738, 7, 87, 8, 355, 514, 1096, 198, 92, 198, 198, 9979, 25882, 62, 41359, 62, 4221, 15675, 50, 25, 514, 1096, 796, 13108, 26, 198, 2, 58, 9078, 4871, 60, 198, 7249, 7231, 33, 11401, 1391, 198, 50258, 2207, 12342, 25, 21059, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 22330, 198, 50258, 2041, 62, 83, 482, 641, 62, 12685, 12342, 25, 21059, 13912, 27, 10100, 11, 514, 1096, 22330, 198, 50258, 875, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 11, 198, 50258, 2041, 62, 83, 482, 641, 62, 12501, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 11, 198, 50258, 40364, 62, 83, 7278, 25, 38692, 27, 3041, 25636, 22330, 198, 50258, 2041, 62, 260, 25636, 62, 83, 7278, 25, 38692, 27, 3041, 25636, 22330, 198, 50258, 23243, 62, 30001, 62, 33661, 25, 38692, 27, 53, 721, 27, 84, 23, 4211, 11, 198, 92, 198, 198, 23928, 7231, 33, 11401, 1391, 198, 50258, 24714, 4808, 1136, 62, 28781, 62, 260, 25636, 39434, 944, 8, 4613, 1222, 3041, 25636, 1391, 198, 50262, 3373, 4091, 2854, 4710, 2029, 329, 644, 428, 318, 546, 198, 50262, 3373, 632, 338, 635, 257, 1310, 474, 39556, 11, 3387, 787, 257, 1365, 2196, 286, 340, 0, 198, 50262, 3373, 2102, 11, 340, 338, 3621, 326, 428, 1595, 470, 13044, 4088, 284, 1790, 12, 24489, 14390, 198, 50262, 1222, 944, 13, 260, 25636, 62, 83, 7278, 58, 17831, 62, 14421, 62, 16663, 3419, 4064, 25882, 62, 41359, 62, 4221, 15675, 50, 60, 198, 50258, 1782, 628, 50258, 24714, 4808, 1136, 62, 28781, 62, 20887, 62, 260, 25636, 39434, 944, 8, 4613, 1222, 3041, 25636, 1391, 198, 50262, 1222, 944, 13, 20887, 62, 260, 25636, 62, 83, 7278, 58, 17831, 62, 14421, 62, 16663, 3419, 4064, 25882, 62, 41359, 62, 4221, 15675, 50, 60, 198, 50258, 1782, 628, 50258, 24714, 4808, 12501, 1098, 62, 30191, 39434, 944, 11, 16326, 25, 1222, 58, 385, 1096, 12962, 4613, 38692, 27, 84, 23, 29, 1391, 198, 50262, 1309, 4517, 1005, 796, 38692, 3712, 4480, 62, 42404, 7, 83, 482, 641, 13, 11925, 3419, 1635, 362, 1776, 198, 50262, 329, 11241, 287, 16326, 1391, 198, 50266, 1309, 11241, 62, 33661, 796, 2116, 198, 50270, 764, 12501, 12342, 198, 50270, 764, 1136, 7, 30001, 8, 198, 50270, 764, 403, 37150, 62, 273, 62, 17772, 7, 15886, 1222, 944, 13, 20887, 62, 83, 482, 641, 62, 12501, 12342, 58, 30001, 36563, 198, 50266, 1005, 13, 2302, 437, 7, 30001, 62, 33661, 1776, 198, 50262, 1782, 198, 50262, 1005, 198, 50258, 1782, 628, 50258, 24714, 4808, 268, 8189, 62, 35947, 62, 30191, 39434, 944, 11, 2420, 25, 1222, 2536, 8, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 50262, 3373, 770, 318, 262, 4755, 286, 262, 21004, 9156, 26, 262, 584, 5499, 287, 994, 198, 50262, 3373, 655, 787, 1243, 8253, 47226, 198, 50262, 1309, 40364, 796, 2116, 13557, 1136, 62, 28781, 62, 260, 25636, 9783, 198, 50262, 1309, 4517, 1005, 796, 43030, 0, 58, 11208, 198, 50262, 329, 2603, 287, 40364, 13, 19796, 62, 2676, 7, 5239, 8, 1391, 198, 50266, 1309, 3704, 796, 2603, 13, 403, 37150, 22446, 292, 62, 2536, 22446, 292, 62, 33661, 9783, 198, 50266, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 8, 1391, 198, 50270, 1005, 13, 14689, 46491, 30001, 1776, 198, 50270, 2555, 26, 198, 50266, 1782, 198, 50266, 1005, 13, 2302, 437, 39434, 26327, 62, 24874, 62, 268, 8189, 7, 12239, 11, 1222, 944, 13, 12685, 12342, 18125, 198, 50262, 1782, 198, 50262, 1005, 198, 50258, 1782, 628, 50258, 24714, 4808, 268, 8189, 62, 30191, 39434, 944, 11, 2420, 25, 1222, 2536, 11, 3142, 62, 20887, 25, 1222, 26257, 7248, 27, 5, 2536, 43734, 4613, 357, 53, 721, 27, 385, 1096, 22330, 514, 1096, 8, 1391, 198, 50262, 1309, 2041, 62, 260, 25636, 796, 2116, 13557, 1136, 62, 28781, 62, 20887, 62, 260, 25636, 9783, 198, 50262, 1309, 40364, 796, 2116, 13557, 1136, 62, 28781, 62, 260, 25636, 9783, 198, 50262, 1309, 4517, 1005, 796, 43030, 0, 58, 11208, 628, 50262, 1309, 4517, 923, 796, 657, 26, 198, 50262, 1309, 4517, 938, 62, 12239, 62, 30001, 62, 11925, 796, 657, 26, 198, 50262, 9052, 1391, 198, 50266, 1309, 4517, 1306, 62, 20887, 26, 198, 50266, 1309, 4517, 923, 62, 19796, 796, 923, 26, 198, 50266, 9052, 1391, 198, 50270, 3373, 9938, 262, 1306, 3142, 2041, 11241, 11, 611, 597, 198, 50270, 1306, 62, 20887, 796, 2041, 62, 260, 25636, 13, 19796, 62, 6738, 62, 1930, 7, 5239, 11, 923, 62, 19796, 737, 403, 37150, 9783, 198, 50270, 2872, 1306, 62, 20887, 1391, 198, 50274, 2773, 7, 76, 8, 5218, 1391, 198, 50278, 611, 3142, 62, 20887, 13, 3642, 1299, 39434, 5239, 58, 76, 13, 9688, 3419, 492, 76, 13, 437, 3419, 12962, 1391, 198, 50271, 50266, 2270, 26, 198, 50278, 1782, 198, 50278, 923, 62, 19796, 796, 285, 13, 9688, 3419, 1343, 352, 26, 198, 50274, 1782, 198, 50274, 6045, 5218, 2270, 11, 198, 50270, 1782, 198, 50266, 1782, 198, 50266, 1309, 886, 796, 1306, 62, 20887, 13, 8899, 62, 273, 7, 5239, 13, 11925, 22784, 930, 76, 91, 285, 13, 9688, 35430, 628, 50266, 3373, 16805, 11, 994, 356, 467, 11, 8996, 428, 9156, 284, 4808, 268, 8189, 62, 35947, 62, 30191, 198, 50266, 329, 2603, 287, 40364, 13, 19796, 62, 2676, 39434, 5239, 58, 9688, 492, 437, 12962, 1391, 198, 50270, 1309, 3704, 796, 2603, 13, 403, 37150, 22446, 292, 62, 2536, 22446, 292, 62, 33661, 9783, 198, 50270, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 8, 1391, 198, 50274, 938, 62, 12239, 62, 30001, 62, 11925, 796, 352, 26, 198, 50274, 1005, 13, 14689, 46491, 30001, 1776, 198, 50274, 2555, 26, 198, 50270, 1782, 198, 50270, 1309, 16326, 796, 18022, 62, 24874, 62, 268, 8189, 7, 12239, 11, 1222, 944, 13, 12685, 12342, 1776, 198, 50270, 938, 62, 12239, 62, 30001, 62, 11925, 796, 16326, 13, 11925, 9783, 198, 50270, 1005, 13, 2302, 437, 39434, 83, 482, 641, 1776, 198, 50266, 1782, 628, 50266, 2872, 1306, 62, 20887, 1391, 198, 50270, 3373, 843, 994, 356, 4574, 262, 2041, 11241, 198, 50270, 2773, 7, 76, 8, 5218, 1391, 198, 50274, 1309, 3704, 796, 285, 13, 292, 62, 2536, 9783, 198, 50274, 1309, 11241, 796, 2116, 13, 20887, 62, 83, 482, 641, 62, 12685, 12342, 58, 12239, 11208, 198, 50274, 1005, 13, 14689, 7, 30001, 1776, 198, 50274, 923, 796, 285, 13, 437, 9783, 198, 50274, 938, 62, 12239, 62, 30001, 62, 11925, 796, 657, 26, 198, 50270, 1782, 198, 50270, 6045, 5218, 2270, 11, 198, 50266, 1782, 198, 50262, 1782, 628, 50262, 3373, 938, 62, 12239, 62, 30001, 62, 11925, 318, 703, 867, 16326, 1625, 422, 262, 938, 40364, 6626, 13, 770, 318, 973, 198, 50262, 3373, 329, 13213, 21354, 16326, 11, 1201, 345, 460, 470, 20121, 1973, 357, 31284, 8, 40364, 30778, 198, 50262, 357, 1186, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 198, 50258, 1782, 628, 50258, 24714, 4808, 24988, 589, 62, 12957, 62, 12239, 62, 30001, 62, 11925, 7, 198, 50262, 1222, 944, 11, 198, 50262, 16326, 25, 38692, 27, 385, 1096, 22330, 198, 50262, 4517, 938, 62, 12239, 62, 30001, 62, 11925, 25, 514, 1096, 11, 198, 50258, 1267, 4613, 357, 53, 721, 27, 385, 1096, 22330, 514, 1096, 8, 1391, 198, 50262, 3373, 8989, 11, 262, 7064, 810, 674, 40364, 30778, 460, 307, 21354, 13, 198, 50262, 3373, 1114, 262, 4959, 286, 13213, 21354, 16326, 11, 21354, 40364, 26021, 198, 50262, 3373, 318, 691, 257, 1917, 611, 257, 6626, 326, 373, 1944, 27934, 11, 1201, 428, 460, 198, 50262, 3373, 1085, 284, 35981, 286, 16326, 4306, 1807, 284, 307, 8245, 13, 198, 50262, 3373, 537, 3064, 74, 62, 8692, 1838, 674, 1204, 1327, 416, 1390, 262, 3467, 82, 9, 58, 59, 81, 59, 77, 48688, 198, 50262, 3373, 3912, 13, 770, 460, 304, 13, 70, 13, 2728, 37082, 77, 1, 1343, 366, 366, 284, 1716, 37082, 77, 3467, 77, 1911, 198, 50262, 3373, 3423, 318, 257, 2068, 290, 11841, 4259, 25, 198, 50262, 1391, 198, 50266, 1309, 11241, 62, 271, 62, 439, 62, 13200, 796, 930, 30001, 91, 1391, 198, 50270, 2116, 13, 12501, 12342, 198, 50274, 764, 1136, 7, 30001, 8, 198, 50274, 764, 8899, 7, 91, 30001, 62, 33661, 91, 1391, 198, 50278, 11241, 62, 33661, 198, 50271, 50266, 764, 2676, 3419, 198, 50271, 50266, 764, 18218, 3419, 198, 50271, 50266, 764, 439, 7, 91, 5, 65, 91, 685, 65, 6, 46083, 275, 6, 59, 77, 3256, 275, 6, 59, 83, 6, 4083, 3642, 1299, 39434, 65, 4008, 198, 50274, 32092, 198, 50274, 764, 403, 37150, 62, 273, 7, 9562, 8, 198, 50266, 18083, 198, 50266, 611, 938, 62, 12239, 62, 30001, 62, 11925, 1875, 657, 198, 50270, 11405, 11241, 62, 271, 62, 439, 62, 13200, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 12962, 198, 50266, 1391, 198, 50270, 981, 357, 12957, 62, 12239, 62, 30001, 62, 11925, 1279, 16326, 13, 11925, 28955, 198, 50274, 11405, 11241, 62, 271, 62, 439, 62, 13200, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 532, 352, 12962, 198, 50270, 1391, 198, 50274, 938, 62, 12239, 62, 30001, 62, 11925, 15853, 352, 26, 198, 50270, 1782, 198, 50266, 1782, 198, 50262, 1782, 198, 50262, 14257, 62, 30493, 0, 7, 12957, 62, 12239, 62, 30001, 62, 11925, 19841, 16326, 13, 11925, 35430, 628, 50262, 357, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 198, 50258, 1782, 628, 50258, 24714, 4808, 268, 8189, 62, 403, 31284, 62, 30191, 7, 198, 50262, 1222, 944, 11, 198, 50262, 2420, 25, 1222, 2536, 11, 198, 50262, 3142, 62, 20887, 25, 1222, 26257, 7248, 27, 5, 2536, 22330, 198, 50258, 1267, 4613, 357, 53, 721, 27, 385, 1096, 22330, 21059, 7248, 27, 53, 721, 27, 385, 1096, 4211, 8, 1391, 198, 50262, 1309, 357, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 2116, 13557, 268, 8189, 62, 30191, 7, 5239, 11, 3142, 62, 20887, 1776, 198, 50262, 611, 938, 62, 12239, 62, 30001, 62, 11925, 6624, 657, 1391, 198, 50266, 3373, 1002, 938, 62, 12239, 62, 30001, 62, 11925, 318, 6632, 11, 262, 938, 11241, 373, 257, 2041, 11241, 290, 356, 423, 198, 50266, 3373, 645, 21354, 9881, 198, 50266, 1441, 357, 83, 482, 641, 11, 21059, 7248, 3712, 3605, 35430, 198, 50262, 1782, 198, 50262, 1309, 357, 21973, 16326, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 198, 50266, 2116, 13557, 24988, 589, 62, 12957, 62, 12239, 62, 30001, 62, 11925, 7, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 628, 50262, 1309, 21354, 62, 33661, 796, 2116, 13557, 12501, 1098, 62, 30191, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 492, 36563, 198, 50262, 16326, 13, 2213, 19524, 378, 7, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 628, 50262, 3373, 16926, 46, 25, 356, 815, 1949, 7069, 284, 1064, 3224, 8245, 16326, 198, 50262, 3373, 770, 561, 4646, 262, 2033, 286, 1005, 4233, 1710, 618, 13213, 1224, 45240, 198, 50262, 3373, 33973, 284, 262, 9156, 287, 281, 4697, 2196, 286, 428, 2393, 628, 50262, 1309, 4517, 1224, 45240, 796, 21059, 7248, 3712, 3605, 9783, 198, 50262, 611, 21354, 62, 33661, 13, 271, 62, 28920, 3419, 1391, 198, 50266, 1441, 357, 83, 482, 641, 11, 1224, 45240, 1776, 198, 50262, 1782, 628, 50262, 3373, 770, 318, 262, 2562, 1643, 13, 2329, 1064, 477, 2060, 16326, 326, 923, 351, 21354, 62, 33661, 198, 50262, 3373, 357, 8201, 16326, 326, 3446, 2872, 21354, 62, 33661, 8, 198, 50262, 3373, 8621, 283, 803, 428, 422, 262, 9052, 2174, 5419, 351, 2854, 287, 257, 2219, 1339, 13, 198, 50262, 1309, 4517, 966, 796, 2116, 198, 50266, 764, 82, 9741, 62, 30001, 62, 33661, 198, 50266, 764, 3911, 653, 62, 4122, 7, 91, 87, 91, 2124, 13, 292, 62, 48369, 3419, 1279, 21354, 62, 33661, 13, 292, 62, 48369, 35430, 198, 50262, 981, 966, 1279, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 13, 11925, 3419, 198, 50266, 11405, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 301, 5889, 62, 4480, 39434, 403, 31284, 62, 33661, 8, 198, 50262, 1391, 198, 50266, 1224, 45240, 13, 28463, 7, 35138, 0, 58, 198, 50270, 2116, 13, 12685, 12342, 58, 944, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 292, 62, 48369, 3419, 4357, 198, 50266, 2361, 1776, 198, 50266, 966, 15853, 352, 26, 198, 50262, 1782, 628, 50262, 3373, 2735, 4174, 772, 517, 33908, 2700, 13, 1629, 790, 357, 847, 8, 1744, 2292, 329, 262, 965, 2860, 1359, 198, 50262, 3373, 11241, 11, 1673, 36686, 378, 3224, 9881, 422, 326, 11241, 357, 361, 597, 8, 284, 21354, 62, 33661, 11, 198, 50262, 3373, 290, 1005, 4233, 786, 262, 2187, 1517, 290, 766, 644, 356, 651, 13, 198, 50262, 329, 1312, 287, 352, 492, 403, 31284, 62, 33661, 13, 11925, 3419, 1391, 198, 50266, 1309, 21231, 796, 1222, 403, 31284, 62, 33661, 58, 492, 72, 11208, 198, 50266, 1309, 35488, 796, 1222, 403, 31284, 62, 33661, 58, 72, 492, 11208, 198, 50266, 1309, 4517, 966, 796, 2116, 198, 50270, 764, 82, 9741, 62, 30001, 62, 33661, 198, 50270, 764, 3911, 653, 62, 4122, 7, 91, 87, 91, 2124, 13, 292, 62, 48369, 3419, 1279, 35488, 1776, 198, 50266, 3373, 16926, 46, 25, 2448, 69, 6436, 5612, 611, 35488, 4940, 351, 366, 366, 30, 198, 50266, 981, 966, 1279, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 13, 11925, 3419, 198, 50270, 11405, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 301, 5889, 62, 4480, 7, 37333, 844, 8, 198, 50266, 1391, 198, 50270, 1309, 5885, 796, 685, 40290, 11, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 292, 62, 48369, 3419, 4083, 1102, 9246, 9783, 198, 50270, 1309, 30240, 796, 2872, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 39434, 79, 43691, 8, 1391, 198, 50274, 3373, 3461, 453, 11, 428, 318, 18022, 62, 24874, 62, 268, 8189, 39434, 79, 43691, 11, 1222, 944, 13, 12685, 12342, 8, 198, 50274, 3373, 887, 356, 1244, 423, 5495, 257, 40364, 6626, 543, 561, 2948, 4017, 3212, 13, 198, 50274, 3373, 357, 31722, 1744, 287, 262, 4931, 286, 21354, 40364, 30778, 8, 198, 50274, 3373, 1406, 10385, 284, 41002, 12, 23, 290, 466, 40364, 26021, 13, 198, 50274, 3373, 412, 13, 70, 13, 351, 537, 3064, 74, 62, 8692, 366, 220, 220, 2474, 3011, 6626, 284, 366, 366, 1343, 366, 5145, 1600, 198, 50274, 3373, 475, 18022, 62, 24874, 62, 268, 8189, 7203, 220, 220, 2474, 8, 14512, 18022, 62, 24874, 62, 268, 8189, 7203, 366, 8, 198, 50274, 6762, 7, 82, 8, 5218, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 82, 828, 628, 50274, 3373, 5429, 1146, 11, 1771, 393, 407, 428, 3211, 318, 3376, 8338, 319, 1771, 612, 198, 50274, 3373, 561, 307, 257, 40364, 6626, 878, 262, 41002, 12, 23, 40122, 341, 966, 13, 198, 50274, 3373, 18578, 21404, 1576, 326, 645, 530, 481, 1683, 4003, 357, 8499, 477, 11, 661, 1422, 470, 198, 50274, 3373, 4003, 477, 262, 1263, 10421, 287, 262, 2180, 21354, 11241, 7822, 8, 198, 50274, 41512, 28264, 8, 5218, 18022, 62, 24874, 62, 268, 8189, 39434, 79, 43691, 11, 1222, 944, 13, 12685, 12342, 828, 198, 50274, 3373, 13742, 588, 262, 1708, 318, 19827, 475, 11491, 25, 198, 50274, 3373, 41512, 7, 68, 8, 5218, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 13271, 8635, 1391, 198, 50274, 3373, 50259, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 62, 403, 26752, 39434, 79, 43691, 58, 492, 68, 13, 12102, 62, 929, 62, 1462, 3419, 12962, 198, 50274, 3373, 1782, 828, 198, 50270, 18083, 198, 50270, 1309, 4517, 33756, 796, 38692, 3712, 3605, 9783, 198, 50270, 1309, 4517, 33756, 62, 11925, 796, 657, 26, 198, 50270, 329, 11241, 287, 30240, 1391, 198, 50274, 33756, 13, 14689, 7, 30001, 1776, 198, 50274, 33756, 62, 11925, 15853, 2116, 13, 12501, 12342, 58, 5, 30001, 4083, 11925, 9783, 198, 50274, 611, 33756, 62, 11925, 18189, 21354, 62, 33661, 13, 11925, 3419, 1391, 198, 50278, 2270, 26, 198, 50274, 1782, 198, 50270, 1782, 198, 50270, 1224, 45240, 13, 28463, 7, 41068, 1776, 198, 50270, 966, 15853, 352, 26, 198, 50266, 1782, 198, 50262, 1782, 628, 50262, 3373, 770, 318, 635, 407, 15836, 13, 2893, 356, 4143, 7048, 326, 40364, 30778, 389, 8245, 11, 198, 50262, 3373, 12716, 11, 484, 389, 407, 13, 1320, 318, 11, 611, 4375, 9881, 547, 284, 787, 257, 6626, 1656, 287, 198, 50262, 3373, 21354, 62, 33661, 11, 428, 714, 787, 16326, 1744, 543, 674, 9156, 561, 4306, 892, 198, 50262, 3373, 561, 307, 23791, 13, 198, 50262, 3373, 1114, 1672, 11, 351, 308, 457, 17, 11, 262, 779, 286, 3467, 82, 33747, 12248, 59, 50, 8, 1724, 326, 37082, 77, 59, 77, 1, 714, 198, 50262, 3373, 1205, 257, 6626, 11, 304, 13, 70, 13, 37082, 77, 59, 77, 15, 1, 30778, 656, 37082, 77, 1, 10, 1, 59, 77, 1, 10, 1, 15, 1600, 1642, 37082, 77, 1, 257, 1744, 11241, 13, 198, 50262, 3373, 3423, 318, 257, 2068, 290, 11841, 4259, 25, 198, 50262, 3373, 770, 2125, 470, 826, 611, 356, 1683, 4781, 3467, 82, 33747, 12248, 59, 50, 8, 198, 50262, 611, 21354, 62, 33661, 13, 11925, 3419, 1875, 352, 1391, 198, 50266, 1309, 938, 62, 12501, 9043, 796, 275, 2536, 3712, 12501, 1098, 62, 12957, 62, 40477, 23, 7, 403, 31284, 62, 33661, 13, 292, 62, 48369, 35430, 198, 50266, 611, 21354, 62, 33661, 13, 11925, 3419, 532, 938, 62, 12501, 9043, 13, 16, 1875, 657, 198, 50270, 11405, 938, 62, 12501, 9043, 13, 15, 13, 8899, 62, 273, 7, 9562, 11, 930, 66, 91, 269, 13, 271, 62, 1929, 2737, 10223, 28955, 198, 50266, 1391, 198, 50270, 1309, 4517, 302, 12685, 9043, 796, 18022, 62, 24874, 62, 268, 8189, 7, 198, 50274, 1222, 403, 31284, 62, 33661, 58, 492, 403, 31284, 62, 33661, 13, 11925, 3419, 532, 938, 62, 12501, 9043, 13, 16, 4357, 198, 50274, 1222, 944, 13, 12685, 12342, 11, 198, 50270, 5619, 198, 50270, 302, 12685, 9043, 13, 2302, 437, 7, 26327, 62, 24874, 62, 268, 8189, 7, 198, 50274, 1222, 403, 31284, 62, 33661, 58, 403, 31284, 62, 33661, 13, 11925, 3419, 532, 938, 62, 12501, 9043, 13, 16, 492, 4357, 198, 50274, 1222, 944, 13, 12685, 12342, 11, 198, 50270, 29226, 198, 50270, 1224, 45240, 13, 28463, 7, 1361, 40976, 1776, 198, 50266, 1782, 198, 50262, 1782, 628, 50262, 357, 83, 482, 641, 11, 1224, 45240, 8, 198, 50258, 1782, 198, 92, 198, 198, 2, 58, 9078, 24396, 82, 60, 198, 23928, 7231, 33, 11401, 1391, 198, 50258, 1303, 58, 3605, 60, 198, 50258, 24714, 649, 7, 198, 50262, 2207, 12342, 25, 21059, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 22330, 198, 50262, 2041, 62, 83, 482, 641, 62, 12685, 12342, 25, 21059, 13912, 27, 10100, 11, 514, 1096, 22330, 198, 50262, 3912, 25, 1222, 2536, 11, 198, 50258, 1267, 4613, 9485, 23004, 27, 24704, 29, 1391, 198, 50262, 1309, 40364, 796, 797, 25636, 3712, 3605, 7, 33279, 8, 198, 50266, 764, 8899, 62, 8056, 7, 91, 68, 91, 9485, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 11395, 12331, 11, 4808, 33994, 68, 13, 1462, 62, 8841, 3419, 4008, 30, 26, 628, 50262, 1309, 2041, 62, 260, 25636, 796, 1391, 198, 50266, 1309, 4808, 42632, 796, 2041, 62, 83, 482, 641, 62, 12685, 12342, 198, 50270, 764, 13083, 3419, 198, 50270, 764, 8899, 7, 91, 82, 91, 14996, 62, 260, 25636, 3712, 41915, 7, 82, 4008, 198, 50270, 764, 33327, 3712, 27, 53, 721, 27, 62, 4211, 9783, 198, 50266, 797, 25636, 3712, 3605, 39434, 62, 42632, 13, 22179, 7203, 91, 48774, 198, 50270, 764, 8899, 62, 8056, 7, 91, 68, 91, 9485, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 11395, 12331, 11, 4808, 33994, 68, 13, 1462, 62, 8841, 3419, 4008, 30, 198, 50262, 18083, 628, 50262, 1309, 875, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 796, 198, 50266, 2207, 12342, 13, 2676, 22446, 8899, 7, 91, 7, 74, 11, 410, 14726, 20789, 85, 11, 479, 13, 21018, 28955, 737, 33327, 9783, 628, 50262, 6818, 0, 7, 12685, 12342, 13, 11925, 3419, 6624, 875, 12342, 13, 11925, 35430, 628, 50262, 1309, 2041, 62, 83, 482, 641, 62, 12501, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 796, 2041, 62, 83, 482, 641, 62, 12685, 12342, 198, 50266, 764, 2676, 3419, 198, 50266, 764, 8899, 7, 91, 7, 74, 11, 410, 14726, 20789, 85, 11, 479, 13, 292, 62, 33661, 22446, 1462, 62, 35138, 3419, 4008, 198, 50266, 764, 33327, 9783, 628, 50262, 3373, 30698, 780, 314, 836, 470, 760, 703, 284, 1560, 17103, 314, 1101, 407, 1016, 284, 1487, 262, 3975, 198, 50262, 1309, 4517, 23243, 62, 30001, 62, 33661, 25, 38692, 27, 53, 721, 27, 84, 23, 4211, 796, 2207, 12342, 13, 13083, 22446, 565, 12004, 22446, 33327, 9783, 198, 50262, 23243, 62, 30001, 62, 33661, 13, 30619, 9783, 628, 50262, 6762, 7, 14055, 33, 11401, 1391, 198, 50266, 2207, 12342, 11, 198, 50266, 2041, 62, 83, 482, 641, 62, 12685, 12342, 11, 198, 50266, 875, 12342, 11, 198, 50266, 2041, 62, 83, 482, 641, 62, 12501, 12342, 11, 198, 50266, 40364, 62, 83, 7278, 25, 357, 15, 492, 22921, 62, 41359, 62, 4221, 15675, 50, 737, 8899, 7, 91, 62, 91, 40364, 13, 21018, 3419, 737, 33327, 22784, 198, 50266, 2041, 62, 260, 25636, 62, 83, 7278, 25, 357, 15, 492, 22921, 62, 41359, 62, 4221, 15675, 50, 8, 198, 50270, 764, 8899, 7, 91, 62, 91, 2041, 62, 260, 25636, 13, 21018, 28955, 198, 50270, 764, 33327, 22784, 198, 50266, 23243, 62, 30001, 62, 33661, 11, 198, 50262, 32092, 198, 50258, 1782, 628, 50258, 3373, 36658, 18604, 198, 50258, 3373, 14711, 7656, 198, 50258, 3373, 36658, 18604, 628, 50258, 24714, 37773, 62, 35947, 39434, 944, 11, 12972, 25, 11361, 11, 2420, 25, 1222, 2536, 8, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 50262, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 5239, 4008, 198, 50258, 1782, 628, 50258, 24714, 37773, 39434, 944, 11, 12972, 25, 11361, 11, 2420, 25, 1222, 2536, 11, 3142, 62, 20887, 25, 21059, 7248, 27, 5, 2536, 43734, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 50262, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 268, 8189, 62, 30191, 7, 5239, 11, 1222, 40845, 62, 20887, 737, 15, 8, 198, 50258, 1782, 628, 50258, 24714, 4808, 268, 8189, 62, 33661, 39434, 944, 11, 12972, 25, 11361, 11, 9881, 25, 1222, 58, 84, 23, 12962, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 50262, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 1391, 198, 50266, 2872, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 7, 33661, 8, 1391, 198, 50270, 6762, 7, 5239, 8, 5218, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 5239, 828, 198, 50270, 41512, 7, 68, 8, 5218, 1391, 198, 50274, 1309, 2420, 796, 21596, 1391, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 62, 403, 26752, 39434, 33661, 58, 492, 68, 13, 12102, 62, 929, 62, 1462, 3419, 12962, 18083, 198, 50274, 1309, 357, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 2116, 13557, 268, 8189, 62, 30191, 7, 5239, 11, 1222, 26257, 7248, 3712, 3605, 35430, 198, 50274, 1309, 357, 21973, 16326, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 198, 50278, 2116, 13557, 24988, 589, 62, 12957, 62, 12239, 62, 30001, 62, 11925, 7, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 198, 50274, 611, 5145, 83, 482, 641, 13, 271, 62, 28920, 3419, 11405, 938, 62, 12239, 62, 30001, 62, 11925, 1875, 657, 1391, 198, 50278, 3373, 406, 404, 572, 262, 16326, 422, 262, 938, 3704, 290, 1057, 347, 11401, 319, 262, 5637, 9881, 198, 50278, 3373, 32499, 5183, 21404, 11, 475, 428, 743, 407, 307, 3376, 611, 356, 1549, 423, 550, 257, 40364, 198, 50278, 3373, 6626, 1022, 262, 4938, 41002, 12, 23, 290, 262, 12515, 9881, 11, 543, 318, 1521, 428, 198, 50278, 3373, 2446, 318, 2839, 198, 50278, 1309, 4517, 21354, 62, 33661, 796, 198, 50271, 50266, 2116, 13557, 12501, 1098, 62, 30191, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 492, 36563, 198, 50278, 21354, 62, 33661, 13, 2302, 437, 62, 6738, 62, 48369, 39434, 33661, 58, 68, 13, 12102, 62, 929, 62, 1462, 3419, 492, 36563, 628, 50278, 16326, 13, 2213, 19524, 378, 7, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 198, 50278, 16326, 13, 2302, 437, 7, 26327, 62, 24874, 62, 268, 8189, 39434, 403, 31284, 62, 33661, 11, 1222, 944, 13, 12685, 12342, 18125, 198, 50274, 1782, 198, 50274, 16326, 198, 50270, 1782, 198, 50266, 1782, 198, 50262, 32092, 198, 50258, 1782, 628, 50258, 24714, 37773, 62, 4480, 62, 403, 31284, 7, 198, 50262, 1222, 944, 11, 198, 50262, 12972, 25, 11361, 11, 198, 50262, 2420, 25, 1222, 2536, 11, 198, 50262, 3142, 62, 20887, 25, 21059, 7248, 27, 5, 2536, 22330, 198, 50258, 1267, 4613, 9485, 27, 20519, 51, 29291, 29, 1391, 198, 50262, 1309, 357, 83, 482, 641, 11, 1224, 45240, 8, 796, 198, 50266, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 268, 8189, 62, 403, 31284, 62, 30191, 7, 5239, 11, 1222, 40845, 62, 20887, 18125, 198, 50262, 1309, 12972, 62, 785, 37069, 507, 796, 198, 50266, 9485, 8053, 3712, 3605, 7, 9078, 11, 1224, 45240, 13, 2676, 22446, 8899, 7, 91, 41068, 91, 9485, 8053, 3712, 3605, 7, 9078, 11, 1222, 41068, 58, 492, 60, 4008, 1776, 198, 50262, 357, 83, 482, 641, 11, 12972, 62, 785, 37069, 507, 737, 20424, 62, 9078, 7, 9078, 8, 198, 50258, 1782, 628, 50258, 24714, 37773, 62, 29762, 62, 30001, 39434, 944, 11, 3704, 25, 1222, 58, 84, 23, 12962, 4613, 9485, 23004, 27, 385, 1096, 29, 1391, 198, 50262, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 737, 22163, 798, 3419, 1391, 198, 50266, 1441, 6762, 7, 30001, 1776, 198, 50262, 1782, 198, 50262, 611, 1309, 6762, 7, 12239, 62, 2536, 8, 796, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 7, 12239, 8, 1391, 198, 50266, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 20887, 62, 83, 482, 641, 62, 12685, 12342, 13, 1136, 7, 12239, 62, 2536, 737, 22163, 798, 3419, 1391, 198, 50270, 1441, 6762, 7, 30001, 1776, 198, 50266, 1782, 198, 50262, 1782, 198, 50262, 41512, 7, 20519, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 9218, 12331, 11, 4808, 33994, 12239, 13, 1462, 62, 11990, 3419, 4008, 198, 50258, 1782, 628, 50258, 24714, 37773, 62, 29762, 62, 12239, 39434, 944, 11, 3704, 25, 1222, 58, 84, 23, 12962, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 50262, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 8, 1391, 198, 50266, 1441, 43030, 0, 58, 9, 30001, 11208, 198, 50262, 1782, 198, 50262, 18022, 62, 24874, 62, 268, 8189, 7, 12239, 11, 1222, 944, 13, 12685, 12342, 8, 198, 50258, 1782, 628, 50258, 3373, 36658, 18604, 198, 50258, 3373, 4280, 7656, 198, 50258, 3373, 36658, 18604, 628, 50258, 24714, 36899, 62, 33661, 39434, 944, 11, 12972, 25, 11361, 11, 16326, 25, 38692, 27, 385, 1096, 43734, 4613, 9485, 27, 20519, 45992, 29, 1391, 198, 50262, 1309, 9881, 796, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 12501, 1098, 62, 30191, 39434, 83, 482, 641, 18125, 198, 50262, 9485, 45992, 3712, 3605, 7, 9078, 11, 1222, 33661, 737, 20424, 3419, 198, 50258, 1782, 628, 50258, 24714, 36899, 62, 29762, 62, 30001, 62, 33661, 39434, 944, 11, 12972, 25, 11361, 11, 11241, 25, 514, 1096, 8, 4613, 9485, 23004, 27, 20519, 27, 20519, 45992, 4211, 1391, 198, 50262, 611, 1309, 2773, 7, 33661, 8, 796, 2116, 13, 12501, 12342, 13, 1136, 39434, 30001, 8, 1391, 198, 50266, 1441, 6762, 7, 20519, 45992, 3712, 3605, 7, 9078, 11, 9881, 737, 20424, 35430, 198, 50262, 1782, 198, 50262, 611, 1309, 2773, 7, 33661, 8, 796, 2116, 13, 20887, 62, 83, 482, 641, 62, 12501, 12342, 13, 1136, 39434, 30001, 8, 1391, 198, 50266, 1441, 6762, 7, 20519, 45992, 3712, 3605, 7, 9078, 11, 9881, 737, 20424, 35430, 198, 50262, 1782, 198, 50262, 41512, 7, 20519, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 9218, 12331, 11, 4808, 33994, 30001, 13, 1462, 62, 8841, 3419, 4008, 198, 50258, 1782, 628, 50258, 3373, 36658, 18604, 198, 50258, 3373, 46253, 198, 50258, 3373, 36658, 18604, 628, 50258, 24714, 11241, 62, 26327, 62, 27160, 39434, 944, 11, 12972, 25, 11361, 8, 4613, 38692, 27, 20519, 27, 20519, 45992, 4211, 1391, 198, 50262, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 198, 50266, 764, 2676, 3419, 198, 50266, 764, 8899, 7, 91, 87, 91, 9485, 45992, 3712, 3605, 7, 9078, 11, 2124, 737, 20424, 28955, 198, 50266, 764, 33327, 3419, 198, 50258, 1782, 198, 92, 198, 198, 2, 58, 9078, 21412, 60, 198, 22184, 4808, 83, 1134, 30001, 28264, 9078, 25, 11361, 11, 285, 25, 1222, 20519, 26796, 8, 4613, 9485, 23004, 27, 3419, 29, 1391, 198, 50258, 285, 13, 2860, 62, 4871, 3712, 27, 14055, 33, 11401, 29, 3419, 30, 26, 198, 50258, 6762, 7, 28955, 198, 92, 198, 198, 2, 58, 37581, 7, 9288, 15437, 198, 4666, 5254, 1391, 198, 50258, 779, 17000, 66, 62, 17831, 3712, 37, 87, 26257, 13912, 355, 21059, 13912, 26, 628, 50258, 779, 27021, 3712, 26327, 62, 24874, 62, 35312, 26, 628, 50258, 1303, 58, 9288, 60, 198, 50258, 24714, 845, 62, 36439, 62, 9288, 3419, 1391, 198, 50262, 1309, 4517, 9803, 796, 21059, 13912, 3712, 12286, 9783, 198, 50262, 9803, 13, 28463, 7, 65, 1, 397, 1911, 1462, 62, 35138, 22784, 352, 1776, 198, 50262, 9803, 13, 28463, 7, 65, 1, 10210, 1911, 1462, 62, 35138, 22784, 362, 1776, 628, 50262, 1309, 581, 796, 18022, 62, 24874, 62, 35312, 7, 65, 1, 397, 10210, 1600, 1222, 81, 2283, 1776, 198, 50262, 6818, 62, 27363, 0, 7, 411, 11, 43030, 0, 58, 65, 1, 397, 1600, 275, 1, 10210, 8973, 1776, 198, 50258, 1782, 198, 92, 198] \ No newline at end of file diff --git a/test/Microsoft.ML.Tokenizers.Tests/Data/tokens_p50k_edit.json b/test/Microsoft.ML.Tokenizers.Tests/Data/tokens_p50k_edit.json new file mode 100644 index 0000000000..54efe06336 --- /dev/null +++ b/test/Microsoft.ML.Tokenizers.Tests/Data/tokens_p50k_edit.json @@ -0,0 +1 @@ +[1003, 770, 2198, 318, 649, 290, 2331, 46542, 357, 39363, 351, 9485, 46, 18, 10375, 8, 198, 2, 0, 58, 12154, 7, 565, 41214, 3712, 2865, 808, 62, 67, 567, 69, 62, 5420, 15437, 198, 198, 1904, 14367, 3712, 4033, 26448, 3712, 26257, 7248, 26, 198, 1904, 14367, 3712, 16663, 26, 198, 198, 1904, 14996, 62, 260, 25636, 3712, 3041, 25636, 26, 198, 1904, 279, 8226, 18, 3712, 1069, 11755, 26, 198, 1904, 279, 8226, 18, 3712, 79, 2411, 2507, 3712, 9, 26, 198, 1904, 279, 8226, 18, 3712, 19199, 3712, 90, 20519, 45992, 11, 9485, 8053, 11, 9485, 51, 29291, 19629, 198, 1904, 279, 8226, 18, 3712, 20519, 23004, 26, 198, 1904, 17000, 66, 62, 17831, 3712, 37, 87, 26257, 13912, 355, 21059, 13912, 26, 198, 198, 22184, 4808, 26327, 62, 24874, 62, 647, 469, 27, 51, 33994, 198, 50258, 3704, 25, 1222, 58, 84, 23, 4357, 198, 50258, 9803, 25, 1222, 26257, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 22330, 198, 50258, 277, 25, 4114, 37481, 7, 19282, 3712, 2840, 3712, 17257, 27, 385, 1096, 43734, 4613, 309, 11, 198, 8, 4613, 38692, 27, 51, 29, 1391, 198, 50258, 3373, 770, 318, 257, 15879, 286, 357, 9688, 11, 4279, 737, 198, 50258, 3373, 383, 4279, 318, 286, 262, 18022, 5166, 3599, 379, 2292, 923, 13, 198, 50258, 3373, 383, 4279, 286, 262, 938, 2378, 287, 262, 15879, 318, 407, 257, 4938, 1988, 13, 198, 50258, 1309, 4517, 3354, 25, 38692, 27, 7, 385, 1096, 11, 514, 1096, 8, 29, 796, 357, 15, 492, 12239, 13, 11925, 3419, 1343, 352, 737, 8899, 7, 91, 72, 91, 357, 72, 11, 514, 1096, 3712, 22921, 29720, 33327, 9783, 628, 50258, 1309, 651, 62, 43027, 796, 1391, 198, 50262, 1303, 58, 45145, 7, 33770, 15437, 198, 50262, 930, 42632, 25, 1222, 53, 721, 27, 7, 385, 1096, 11, 514, 1096, 8, 22330, 923, 62, 312, 87, 25, 514, 1096, 11, 14267, 25, 514, 1096, 91, 1391, 198, 50266, 611, 357, 9688, 62, 312, 87, 1343, 14267, 1343, 362, 8, 1279, 3354, 13, 11925, 3419, 1391, 198, 50270, 9803, 198, 50274, 764, 1136, 39434, 12239, 58, 42632, 58, 9688, 62, 312, 87, 4083, 15, 492, 42632, 58, 9688, 62, 312, 87, 1343, 14267, 1343, 362, 4083, 15, 12962, 198, 50274, 764, 22163, 798, 3419, 198, 50266, 1782, 2073, 1391, 198, 50270, 6045, 198, 50266, 1782, 198, 50262, 1782, 198, 50258, 18083, 628, 50258, 3373, 775, 804, 510, 262, 9803, 1752, 287, 262, 3726, 290, 11629, 9404, 4296, 198, 50258, 3373, 606, 1141, 1123, 20121, 11, 543, 12850, 262, 1271, 286, 4279, 804, 4739, 13, 198, 50258, 329, 1312, 287, 657, 492, 42632, 13, 11925, 3419, 532, 362, 1391, 198, 50262, 2872, 651, 62, 43027, 39434, 42632, 11, 1312, 11, 657, 8, 1391, 198, 50266, 2773, 7, 43027, 8, 5218, 1391, 198, 50270, 3373, 514, 1096, 3712, 22921, 318, 257, 1908, 20538, 1988, 290, 2314, 307, 257, 4938, 4279, 198, 50270, 14257, 62, 30493, 0, 7, 43027, 14512, 514, 1096, 3712, 22921, 1776, 198, 50270, 3354, 58, 72, 4083, 16, 796, 4279, 26, 198, 50266, 1782, 198, 50266, 6045, 5218, 1391, 198, 50270, 2555, 26, 198, 50266, 1782, 198, 50262, 18083, 198, 50258, 1782, 628, 50258, 3373, 1002, 345, 423, 299, 3354, 290, 285, 4017, 3212, 11, 428, 857, 440, 7, 10295, 8, 670, 13, 198, 50258, 3373, 775, 714, 466, 1223, 351, 257, 24575, 290, 466, 440, 7, 76, 2604, 299, 8, 670, 13, 198, 50258, 3373, 632, 318, 1593, 284, 2074, 326, 299, 318, 1690, 1402, 38155, 3064, 828, 290, 355, 884, 198, 50258, 3373, 262, 12940, 12, 17946, 1483, 4034, 31405, 262, 8385, 9383, 13357, 21838, 1460, 198, 50258, 3373, 286, 262, 4600, 42632, 63, 15879, 1366, 4645, 2029, 13, 628, 50258, 3373, 5740, 326, 356, 12234, 9881, 11, 407, 11241, 14729, 13, 1081, 890, 355, 356, 4512, 347, 11401, 262, 835, 356, 198, 50258, 3373, 3058, 466, 11, 428, 318, 7548, 13, 1052, 2562, 835, 284, 2270, 428, 561, 307, 284, 875, 43846, 198, 50258, 3373, 20121, 8475, 422, 11241, 6376, 393, 284, 2948, 2176, 11241, 4017, 3212, 13, 198, 50258, 9052, 1391, 198, 50262, 611, 3354, 13, 11925, 3419, 6624, 352, 1391, 198, 50266, 2270, 26, 198, 50262, 1782, 628, 50262, 3373, 514, 1096, 3712, 22921, 318, 257, 1908, 20538, 4279, 1988, 5086, 514, 284, 198, 50262, 3373, 1011, 262, 949, 517, 2952, 198, 50262, 1309, 4517, 949, 62, 43027, 25, 357, 385, 1096, 11, 514, 1096, 8, 796, 357, 385, 1096, 3712, 22921, 11, 657, 1776, 198, 50262, 329, 357, 72, 11, 1222, 28264, 11, 4279, 4008, 287, 3354, 58, 492, 42632, 13, 11925, 3419, 532, 352, 4083, 2676, 22446, 268, 6975, 378, 3419, 1391, 198, 50266, 611, 4279, 1279, 949, 62, 43027, 13, 15, 1391, 198, 50270, 949, 62, 43027, 796, 357, 43027, 11, 1312, 1776, 198, 50266, 1782, 198, 50262, 1782, 628, 50262, 611, 949, 62, 43027, 13, 15, 14512, 514, 1096, 3712, 22921, 1391, 198, 50266, 1309, 1312, 796, 949, 62, 43027, 13, 16, 26, 628, 50266, 3373, 24550, 25, 775, 389, 546, 284, 4781, 3354, 58, 72, 1343, 352, 4083, 775, 466, 407, 466, 340, 198, 50266, 3373, 1865, 780, 612, 389, 12940, 12, 17946, 1483, 4034, 284, 19698, 198, 50266, 3373, 3354, 58, 72, 60, 290, 3354, 58, 72, 12, 16, 60, 878, 10829, 11, 543, 714, 5636, 1077, 198, 50266, 3373, 262, 12940, 13, 6660, 11, 356, 4296, 262, 4279, 17952, 416, 31017, 625, 198, 50266, 3373, 3354, 58, 72, 1343, 352, 4357, 416, 39744, 4600, 1136, 62, 43027, 0, 63, 351, 4600, 48267, 796, 352, 44646, 198, 50266, 3354, 58, 72, 4083, 16, 796, 651, 62, 43027, 39434, 42632, 11, 1312, 11, 352, 737, 403, 37150, 62, 273, 7, 385, 1096, 3712, 22921, 1776, 198, 50266, 611, 1312, 1875, 657, 1391, 198, 50270, 3354, 58, 72, 532, 352, 4083, 16, 796, 651, 62, 43027, 39434, 42632, 11, 1312, 532, 352, 11, 352, 737, 403, 37150, 62, 273, 7, 385, 1096, 3712, 22921, 1776, 198, 50266, 1782, 628, 50266, 3354, 13, 28956, 7, 72, 1343, 352, 1776, 198, 50262, 1782, 2073, 1391, 198, 50266, 2270, 26, 198, 50262, 1782, 198, 50258, 1782, 198, 50258, 1309, 4517, 503, 25, 38692, 27, 51, 29, 796, 38692, 3712, 4480, 62, 42404, 7, 42632, 13, 11925, 3419, 532, 352, 1776, 198, 50258, 329, 1312, 287, 657, 492, 42632, 13, 11925, 3419, 532, 352, 1391, 198, 50262, 503, 13, 14689, 7, 69, 7, 42632, 58, 72, 4083, 15, 492, 42632, 58, 72, 1343, 352, 4083, 15, 18125, 198, 50258, 1782, 198, 50258, 503, 198, 92, 198, 198, 12984, 24714, 18022, 62, 24874, 62, 268, 8189, 7, 12239, 25, 1222, 58, 84, 23, 4357, 9803, 25, 1222, 26257, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 43734, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 50258, 611, 3704, 13, 11925, 3419, 6624, 352, 1391, 198, 50262, 1441, 43030, 0, 58, 81, 2283, 58, 12239, 60, 11208, 198, 50258, 1782, 198, 50258, 4808, 26327, 62, 24874, 62, 647, 469, 7, 12239, 11, 9803, 11, 930, 79, 91, 9803, 58, 5, 12239, 58, 79, 13, 9688, 492, 79, 13, 437, 11907, 8, 198, 92, 198, 198, 12984, 24714, 18022, 62, 24874, 62, 35312, 27, 6, 64, 33994, 12239, 25, 1222, 6, 64, 685, 84, 23, 4357, 9803, 25, 1222, 26257, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 43734, 4613, 38692, 27, 5, 6, 64, 685, 84, 23, 60, 29, 1391, 198, 50258, 611, 3704, 13, 11925, 3419, 6624, 352, 1391, 198, 50262, 1441, 43030, 0, 58, 12239, 11208, 198, 50258, 1782, 198, 50258, 4808, 26327, 62, 24874, 62, 647, 469, 7, 12239, 11, 9803, 11, 930, 79, 91, 1222, 12239, 58, 79, 13, 9688, 492, 79, 13, 437, 12962, 198, 92, 198, 198, 1003, 26386, 2854, 4710, 25, 198, 1003, 198, 1003, 797, 25636, 198, 1003, 29335, 198, 1003, 4042, 286, 262, 640, 318, 3377, 287, 40364, 13, 383, 16638, 835, 284, 2866, 428, 510, 318, 416, 1262, 1342, 14996, 198, 1003, 40364, 3033, 13, 1114, 4554, 11, 1262, 257, 40364, 21136, 12, 540, 416, 4600, 260, 25636, 63, 27021, 318, 513, 87, 5443, 621, 198, 1003, 262, 6678, 40364, 356, 779, 13, 198, 1003, 198, 1003, 2102, 11, 1813, 326, 356, 821, 1262, 257, 40364, 21136, 12, 540, 416, 4600, 260, 25636, 47671, 612, 2125, 470, 881, 3580, 198, 1003, 1022, 1262, 262, 4600, 260, 25636, 63, 27021, 290, 1262, 262, 4600, 69, 3883, 62, 260, 25636, 63, 27021, 13, 198, 1003, 198, 1003, 1318, 318, 281, 1593, 10375, 1022, 4704, 278, 11, 4600, 260, 25636, 63, 290, 4600, 69, 3883, 62, 260, 25636, 44646, 198, 1003, 1649, 1262, 4600, 69, 3883, 62, 260, 25636, 47671, 356, 2277, 4600, 260, 25636, 13, 19796, 62, 265, 44646, 632, 4962, 503, 326, 428, 5640, 22627, 319, 198, 1003, 617, 4517, 540, 12692, 2272, 2641, 286, 4600, 260, 25636, 44646, 770, 5543, 12847, 2854, 13, 1649, 1262, 8631, 198, 1003, 1468, 4600, 260, 25636, 47671, 356, 836, 470, 2277, 428, 11, 780, 4600, 19796, 62, 2676, 63, 468, 257, 1180, 2438, 3108, 13, 198, 1003, 19809, 25, 3740, 1378, 12567, 13, 785, 14, 11469, 12, 17204, 14, 260, 25636, 14, 2436, 672, 14, 9866, 14, 18973, 13775, 10725, 5222, 13, 9132, 198, 1003, 21836, 11, 262, 835, 356, 651, 1088, 428, 318, 351, 1719, 257, 357, 29471, 8, 4704, 1957, 17271, 286, 262, 40364, 329, 198, 1003, 1123, 4704, 13, 198, 1003, 198, 1003, 14122, 278, 198, 1003, 796, 2559, 198, 1003, 314, 3088, 1262, 4600, 2433, 261, 44646, 632, 2492, 470, 1107, 5443, 621, 1262, 11361, 14390, 290, 13011, 262, 402, 4146, 13, 198, 1003, 1406, 24829, 4600, 2433, 261, 63, 0, 3914, 4704, 954, 3503, 307, 287, 1630, 286, 674, 11361, 2985, 13, 198, 1003, 198, 1003, 327, 8103, 198, 1003, 29335, 855, 198, 1003, 383, 4941, 11241, 5847, 468, 281, 300, 622, 12940, 625, 262, 7548, 286, 4600, 26327, 62, 24874, 62, 268, 8189, 44646, 198, 1003, 19486, 11, 356, 550, 530, 1165, 0, 9170, 340, 11, 356, 547, 691, 29627, 5443, 621, 11361, 13, 198, 1003, 314, 973, 281, 33212, 25392, 284, 1805, 262, 12940, 13, 770, 1422, 470, 1283, 284, 5938, 2060, 40945, 2854, 198, 1003, 33752, 11, 475, 340, 750, 2689, 5021, 12, 16663, 276, 2854, 13, 34097, 306, 11, 340, 3947, 284, 2689, 198, 1003, 5021, 12, 16663, 276, 2854, 772, 618, 314, 691, 550, 7183, 357, 11261, 3077, 314, 32621, 1223, 510, 29865, 198, 1003, 21836, 11, 314, 19169, 326, 356, 714, 651, 5755, 286, 262, 12940, 11, 611, 356, 2190, 262, 900, 286, 16326, 355, 257, 12940, 0, 198, 1003, 2312, 389, 3446, 262, 900, 393, 4017, 3212, 326, 389, 1884, 284, 307, 3024, 13, 843, 783, 356, 836, 470, 423, 284, 892, 198, 1003, 546, 11087, 4517, 1799, 11, 4088, 779, 11, 393, 45973, 13, 198, 1003, 198, 1003, 367, 2140, 198, 1003, 29335, 855, 198, 1003, 775, 779, 376, 87, 26257, 13912, 2427, 286, 262, 3210, 21059, 13912, 13, 770, 318, 3863, 588, 257, 642, 12, 940, 4, 1592, 30, 198, 1003, 383, 1459, 7822, 5645, 510, 1804, 257, 1256, 286, 49544, 286, 9881, 13, 554, 4583, 11, 428, 714, 307, 925, 198, 1003, 284, 307, 49544, 286, 734, 12, 28047, 2374, 286, 493, 82, 11, 543, 3073, 588, 340, 743, 635, 307, 257, 3155, 1411, 5443, 13, 198, 198, 1904, 14367, 3712, 22510, 3712, 15419, 28667, 52, 2414, 26, 198, 12984, 2878, 33482, 16818, 7390, 7, 15419, 28667, 52, 2414, 1776, 198, 198, 22184, 12234, 62, 14421, 62, 16663, 3419, 4613, 514, 1096, 1391, 198, 50258, 3373, 632, 338, 4577, 284, 779, 21596, 621, 284, 779, 37862, 13, 17103, 468, 428, 3621, 334, 2414, 4704, 4686, 3753, 198, 50258, 3373, 326, 2499, 1049, 329, 674, 779, 1339, 286, 14928, 31998, 287, 674, 7177, 13, 8989, 11, 198, 50258, 3373, 340, 338, 2839, 13, 2102, 11, 612, 389, 691, 523, 867, 2842, 345, 460, 12461, 257, 334, 2414, 11, 523, 655, 21595, 1133, 198, 50258, 3373, 3740, 1378, 12567, 13, 785, 14, 11469, 12, 17204, 14, 11469, 14, 37165, 14, 37601, 2670, 198, 50258, 1500, 4808, 25, 685, 84, 23, 26, 807, 60, 796, 685, 15, 26, 14367, 3712, 11883, 3712, 7857, 62, 1659, 3712, 27, 19282, 3712, 16663, 3712, 16818, 7390, 29, 3419, 11208, 198, 50258, 1500, 4808, 25, 685, 84, 23, 26, 807, 60, 796, 685, 15, 26, 14367, 3712, 11883, 3712, 7857, 62, 1659, 3712, 27, 49233, 16818, 7390, 29, 3419, 11208, 198, 50258, 1309, 2124, 796, 21596, 1391, 198, 50262, 14367, 3712, 11883, 3712, 7645, 76, 1133, 3712, 27, 19282, 3712, 16663, 3712, 16818, 7390, 11, 33482, 16818, 7390, 33994, 16663, 3712, 14421, 22446, 312, 3419, 737, 15, 198, 50258, 18083, 198, 50258, 334, 2414, 3712, 6738, 7, 87, 8, 355, 514, 1096, 198, 92, 198, 198, 9979, 25882, 62, 41359, 62, 4221, 15675, 50, 25, 514, 1096, 796, 13108, 26, 198, 2, 58, 9078, 4871, 60, 198, 7249, 7231, 33, 11401, 1391, 198, 50258, 2207, 12342, 25, 21059, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 22330, 198, 50258, 2041, 62, 83, 482, 641, 62, 12685, 12342, 25, 21059, 13912, 27, 10100, 11, 514, 1096, 22330, 198, 50258, 875, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 11, 198, 50258, 2041, 62, 83, 482, 641, 62, 12501, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 11, 198, 50258, 40364, 62, 83, 7278, 25, 38692, 27, 3041, 25636, 22330, 198, 50258, 2041, 62, 260, 25636, 62, 83, 7278, 25, 38692, 27, 3041, 25636, 22330, 198, 50258, 23243, 62, 30001, 62, 33661, 25, 38692, 27, 53, 721, 27, 84, 23, 4211, 11, 198, 92, 198, 198, 23928, 7231, 33, 11401, 1391, 198, 50258, 24714, 4808, 1136, 62, 28781, 62, 260, 25636, 39434, 944, 8, 4613, 1222, 3041, 25636, 1391, 198, 50262, 3373, 4091, 2854, 4710, 2029, 329, 644, 428, 318, 546, 198, 50262, 3373, 632, 338, 635, 257, 1310, 474, 39556, 11, 3387, 787, 257, 1365, 2196, 286, 340, 0, 198, 50262, 3373, 2102, 11, 340, 338, 3621, 326, 428, 1595, 470, 13044, 4088, 284, 1790, 12, 24489, 14390, 198, 50262, 1222, 944, 13, 260, 25636, 62, 83, 7278, 58, 17831, 62, 14421, 62, 16663, 3419, 4064, 25882, 62, 41359, 62, 4221, 15675, 50, 60, 198, 50258, 1782, 628, 50258, 24714, 4808, 1136, 62, 28781, 62, 20887, 62, 260, 25636, 39434, 944, 8, 4613, 1222, 3041, 25636, 1391, 198, 50262, 1222, 944, 13, 20887, 62, 260, 25636, 62, 83, 7278, 58, 17831, 62, 14421, 62, 16663, 3419, 4064, 25882, 62, 41359, 62, 4221, 15675, 50, 60, 198, 50258, 1782, 628, 50258, 24714, 4808, 12501, 1098, 62, 30191, 39434, 944, 11, 16326, 25, 1222, 58, 385, 1096, 12962, 4613, 38692, 27, 84, 23, 29, 1391, 198, 50262, 1309, 4517, 1005, 796, 38692, 3712, 4480, 62, 42404, 7, 83, 482, 641, 13, 11925, 3419, 1635, 362, 1776, 198, 50262, 329, 11241, 287, 16326, 1391, 198, 50266, 1309, 11241, 62, 33661, 796, 2116, 198, 50270, 764, 12501, 12342, 198, 50270, 764, 1136, 7, 30001, 8, 198, 50270, 764, 403, 37150, 62, 273, 62, 17772, 7, 15886, 1222, 944, 13, 20887, 62, 83, 482, 641, 62, 12501, 12342, 58, 30001, 36563, 198, 50266, 1005, 13, 2302, 437, 7, 30001, 62, 33661, 1776, 198, 50262, 1782, 198, 50262, 1005, 198, 50258, 1782, 628, 50258, 24714, 4808, 268, 8189, 62, 35947, 62, 30191, 39434, 944, 11, 2420, 25, 1222, 2536, 8, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 50262, 3373, 770, 318, 262, 4755, 286, 262, 21004, 9156, 26, 262, 584, 5499, 287, 994, 198, 50262, 3373, 655, 787, 1243, 8253, 47226, 198, 50262, 1309, 40364, 796, 2116, 13557, 1136, 62, 28781, 62, 260, 25636, 9783, 198, 50262, 1309, 4517, 1005, 796, 43030, 0, 58, 11208, 198, 50262, 329, 2603, 287, 40364, 13, 19796, 62, 2676, 7, 5239, 8, 1391, 198, 50266, 1309, 3704, 796, 2603, 13, 403, 37150, 22446, 292, 62, 2536, 22446, 292, 62, 33661, 9783, 198, 50266, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 8, 1391, 198, 50270, 1005, 13, 14689, 46491, 30001, 1776, 198, 50270, 2555, 26, 198, 50266, 1782, 198, 50266, 1005, 13, 2302, 437, 39434, 26327, 62, 24874, 62, 268, 8189, 7, 12239, 11, 1222, 944, 13, 12685, 12342, 18125, 198, 50262, 1782, 198, 50262, 1005, 198, 50258, 1782, 628, 50258, 24714, 4808, 268, 8189, 62, 30191, 39434, 944, 11, 2420, 25, 1222, 2536, 11, 3142, 62, 20887, 25, 1222, 26257, 7248, 27, 5, 2536, 43734, 4613, 357, 53, 721, 27, 385, 1096, 22330, 514, 1096, 8, 1391, 198, 50262, 1309, 2041, 62, 260, 25636, 796, 2116, 13557, 1136, 62, 28781, 62, 20887, 62, 260, 25636, 9783, 198, 50262, 1309, 40364, 796, 2116, 13557, 1136, 62, 28781, 62, 260, 25636, 9783, 198, 50262, 1309, 4517, 1005, 796, 43030, 0, 58, 11208, 628, 50262, 1309, 4517, 923, 796, 657, 26, 198, 50262, 1309, 4517, 938, 62, 12239, 62, 30001, 62, 11925, 796, 657, 26, 198, 50262, 9052, 1391, 198, 50266, 1309, 4517, 1306, 62, 20887, 26, 198, 50266, 1309, 4517, 923, 62, 19796, 796, 923, 26, 198, 50266, 9052, 1391, 198, 50270, 3373, 9938, 262, 1306, 3142, 2041, 11241, 11, 611, 597, 198, 50270, 1306, 62, 20887, 796, 2041, 62, 260, 25636, 13, 19796, 62, 6738, 62, 1930, 7, 5239, 11, 923, 62, 19796, 737, 403, 37150, 9783, 198, 50270, 2872, 1306, 62, 20887, 1391, 198, 50274, 2773, 7, 76, 8, 5218, 1391, 198, 50278, 611, 3142, 62, 20887, 13, 3642, 1299, 39434, 5239, 58, 76, 13, 9688, 3419, 492, 76, 13, 437, 3419, 12962, 1391, 198, 50271, 50266, 2270, 26, 198, 50278, 1782, 198, 50278, 923, 62, 19796, 796, 285, 13, 9688, 3419, 1343, 352, 26, 198, 50274, 1782, 198, 50274, 6045, 5218, 2270, 11, 198, 50270, 1782, 198, 50266, 1782, 198, 50266, 1309, 886, 796, 1306, 62, 20887, 13, 8899, 62, 273, 7, 5239, 13, 11925, 22784, 930, 76, 91, 285, 13, 9688, 35430, 628, 50266, 3373, 16805, 11, 994, 356, 467, 11, 8996, 428, 9156, 284, 4808, 268, 8189, 62, 35947, 62, 30191, 198, 50266, 329, 2603, 287, 40364, 13, 19796, 62, 2676, 39434, 5239, 58, 9688, 492, 437, 12962, 1391, 198, 50270, 1309, 3704, 796, 2603, 13, 403, 37150, 22446, 292, 62, 2536, 22446, 292, 62, 33661, 9783, 198, 50270, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 8, 1391, 198, 50274, 938, 62, 12239, 62, 30001, 62, 11925, 796, 352, 26, 198, 50274, 1005, 13, 14689, 46491, 30001, 1776, 198, 50274, 2555, 26, 198, 50270, 1782, 198, 50270, 1309, 16326, 796, 18022, 62, 24874, 62, 268, 8189, 7, 12239, 11, 1222, 944, 13, 12685, 12342, 1776, 198, 50270, 938, 62, 12239, 62, 30001, 62, 11925, 796, 16326, 13, 11925, 9783, 198, 50270, 1005, 13, 2302, 437, 39434, 83, 482, 641, 1776, 198, 50266, 1782, 628, 50266, 2872, 1306, 62, 20887, 1391, 198, 50270, 3373, 843, 994, 356, 4574, 262, 2041, 11241, 198, 50270, 2773, 7, 76, 8, 5218, 1391, 198, 50274, 1309, 3704, 796, 285, 13, 292, 62, 2536, 9783, 198, 50274, 1309, 11241, 796, 2116, 13, 20887, 62, 83, 482, 641, 62, 12685, 12342, 58, 12239, 11208, 198, 50274, 1005, 13, 14689, 7, 30001, 1776, 198, 50274, 923, 796, 285, 13, 437, 9783, 198, 50274, 938, 62, 12239, 62, 30001, 62, 11925, 796, 657, 26, 198, 50270, 1782, 198, 50270, 6045, 5218, 2270, 11, 198, 50266, 1782, 198, 50262, 1782, 628, 50262, 3373, 938, 62, 12239, 62, 30001, 62, 11925, 318, 703, 867, 16326, 1625, 422, 262, 938, 40364, 6626, 13, 770, 318, 973, 198, 50262, 3373, 329, 13213, 21354, 16326, 11, 1201, 345, 460, 470, 20121, 1973, 357, 31284, 8, 40364, 30778, 198, 50262, 357, 1186, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 198, 50258, 1782, 628, 50258, 24714, 4808, 24988, 589, 62, 12957, 62, 12239, 62, 30001, 62, 11925, 7, 198, 50262, 1222, 944, 11, 198, 50262, 16326, 25, 38692, 27, 385, 1096, 22330, 198, 50262, 4517, 938, 62, 12239, 62, 30001, 62, 11925, 25, 514, 1096, 11, 198, 50258, 1267, 4613, 357, 53, 721, 27, 385, 1096, 22330, 514, 1096, 8, 1391, 198, 50262, 3373, 8989, 11, 262, 7064, 810, 674, 40364, 30778, 460, 307, 21354, 13, 198, 50262, 3373, 1114, 262, 4959, 286, 13213, 21354, 16326, 11, 21354, 40364, 26021, 198, 50262, 3373, 318, 691, 257, 1917, 611, 257, 6626, 326, 373, 1944, 27934, 11, 1201, 428, 460, 198, 50262, 3373, 1085, 284, 35981, 286, 16326, 4306, 1807, 284, 307, 8245, 13, 198, 50262, 3373, 537, 3064, 74, 62, 8692, 1838, 674, 1204, 1327, 416, 1390, 262, 3467, 82, 9, 58, 59, 81, 59, 77, 48688, 198, 50262, 3373, 3912, 13, 770, 460, 304, 13, 70, 13, 2728, 37082, 77, 1, 1343, 366, 366, 284, 1716, 37082, 77, 3467, 77, 1911, 198, 50262, 3373, 3423, 318, 257, 2068, 290, 11841, 4259, 25, 198, 50262, 1391, 198, 50266, 1309, 11241, 62, 271, 62, 439, 62, 13200, 796, 930, 30001, 91, 1391, 198, 50270, 2116, 13, 12501, 12342, 198, 50274, 764, 1136, 7, 30001, 8, 198, 50274, 764, 8899, 7, 91, 30001, 62, 33661, 91, 1391, 198, 50278, 11241, 62, 33661, 198, 50271, 50266, 764, 2676, 3419, 198, 50271, 50266, 764, 18218, 3419, 198, 50271, 50266, 764, 439, 7, 91, 5, 65, 91, 685, 65, 6, 46083, 275, 6, 59, 77, 3256, 275, 6, 59, 83, 6, 4083, 3642, 1299, 39434, 65, 4008, 198, 50274, 32092, 198, 50274, 764, 403, 37150, 62, 273, 7, 9562, 8, 198, 50266, 18083, 198, 50266, 611, 938, 62, 12239, 62, 30001, 62, 11925, 1875, 657, 198, 50270, 11405, 11241, 62, 271, 62, 439, 62, 13200, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 12962, 198, 50266, 1391, 198, 50270, 981, 357, 12957, 62, 12239, 62, 30001, 62, 11925, 1279, 16326, 13, 11925, 28955, 198, 50274, 11405, 11241, 62, 271, 62, 439, 62, 13200, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 532, 352, 12962, 198, 50270, 1391, 198, 50274, 938, 62, 12239, 62, 30001, 62, 11925, 15853, 352, 26, 198, 50270, 1782, 198, 50266, 1782, 198, 50262, 1782, 198, 50262, 14257, 62, 30493, 0, 7, 12957, 62, 12239, 62, 30001, 62, 11925, 19841, 16326, 13, 11925, 35430, 628, 50262, 357, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 198, 50258, 1782, 628, 50258, 24714, 4808, 268, 8189, 62, 403, 31284, 62, 30191, 7, 198, 50262, 1222, 944, 11, 198, 50262, 2420, 25, 1222, 2536, 11, 198, 50262, 3142, 62, 20887, 25, 1222, 26257, 7248, 27, 5, 2536, 22330, 198, 50258, 1267, 4613, 357, 53, 721, 27, 385, 1096, 22330, 21059, 7248, 27, 53, 721, 27, 385, 1096, 4211, 8, 1391, 198, 50262, 1309, 357, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 2116, 13557, 268, 8189, 62, 30191, 7, 5239, 11, 3142, 62, 20887, 1776, 198, 50262, 611, 938, 62, 12239, 62, 30001, 62, 11925, 6624, 657, 1391, 198, 50266, 3373, 1002, 938, 62, 12239, 62, 30001, 62, 11925, 318, 6632, 11, 262, 938, 11241, 373, 257, 2041, 11241, 290, 356, 423, 198, 50266, 3373, 645, 21354, 9881, 198, 50266, 1441, 357, 83, 482, 641, 11, 21059, 7248, 3712, 3605, 35430, 198, 50262, 1782, 198, 50262, 1309, 357, 21973, 16326, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 198, 50266, 2116, 13557, 24988, 589, 62, 12957, 62, 12239, 62, 30001, 62, 11925, 7, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 628, 50262, 1309, 21354, 62, 33661, 796, 2116, 13557, 12501, 1098, 62, 30191, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 492, 36563, 198, 50262, 16326, 13, 2213, 19524, 378, 7, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 628, 50262, 3373, 16926, 46, 25, 356, 815, 1949, 7069, 284, 1064, 3224, 8245, 16326, 198, 50262, 3373, 770, 561, 4646, 262, 2033, 286, 1005, 4233, 1710, 618, 13213, 1224, 45240, 198, 50262, 3373, 33973, 284, 262, 9156, 287, 281, 4697, 2196, 286, 428, 2393, 628, 50262, 1309, 4517, 1224, 45240, 796, 21059, 7248, 3712, 3605, 9783, 198, 50262, 611, 21354, 62, 33661, 13, 271, 62, 28920, 3419, 1391, 198, 50266, 1441, 357, 83, 482, 641, 11, 1224, 45240, 1776, 198, 50262, 1782, 628, 50262, 3373, 770, 318, 262, 2562, 1643, 13, 2329, 1064, 477, 2060, 16326, 326, 923, 351, 21354, 62, 33661, 198, 50262, 3373, 357, 8201, 16326, 326, 3446, 2872, 21354, 62, 33661, 8, 198, 50262, 3373, 8621, 283, 803, 428, 422, 262, 9052, 2174, 5419, 351, 2854, 287, 257, 2219, 1339, 13, 198, 50262, 1309, 4517, 966, 796, 2116, 198, 50266, 764, 82, 9741, 62, 30001, 62, 33661, 198, 50266, 764, 3911, 653, 62, 4122, 7, 91, 87, 91, 2124, 13, 292, 62, 48369, 3419, 1279, 21354, 62, 33661, 13, 292, 62, 48369, 35430, 198, 50262, 981, 966, 1279, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 13, 11925, 3419, 198, 50266, 11405, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 301, 5889, 62, 4480, 39434, 403, 31284, 62, 33661, 8, 198, 50262, 1391, 198, 50266, 1224, 45240, 13, 28463, 7, 35138, 0, 58, 198, 50270, 2116, 13, 12685, 12342, 58, 944, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 292, 62, 48369, 3419, 4357, 198, 50266, 2361, 1776, 198, 50266, 966, 15853, 352, 26, 198, 50262, 1782, 628, 50262, 3373, 2735, 4174, 772, 517, 33908, 2700, 13, 1629, 790, 357, 847, 8, 1744, 2292, 329, 262, 965, 2860, 1359, 198, 50262, 3373, 11241, 11, 1673, 36686, 378, 3224, 9881, 422, 326, 11241, 357, 361, 597, 8, 284, 21354, 62, 33661, 11, 198, 50262, 3373, 290, 1005, 4233, 786, 262, 2187, 1517, 290, 766, 644, 356, 651, 13, 198, 50262, 329, 1312, 287, 352, 492, 403, 31284, 62, 33661, 13, 11925, 3419, 1391, 198, 50266, 1309, 21231, 796, 1222, 403, 31284, 62, 33661, 58, 492, 72, 11208, 198, 50266, 1309, 35488, 796, 1222, 403, 31284, 62, 33661, 58, 72, 492, 11208, 198, 50266, 1309, 4517, 966, 796, 2116, 198, 50270, 764, 82, 9741, 62, 30001, 62, 33661, 198, 50270, 764, 3911, 653, 62, 4122, 7, 91, 87, 91, 2124, 13, 292, 62, 48369, 3419, 1279, 35488, 1776, 198, 50266, 3373, 16926, 46, 25, 2448, 69, 6436, 5612, 611, 35488, 4940, 351, 366, 366, 30, 198, 50266, 981, 966, 1279, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 13, 11925, 3419, 198, 50270, 11405, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 301, 5889, 62, 4480, 7, 37333, 844, 8, 198, 50266, 1391, 198, 50270, 1309, 5885, 796, 685, 40290, 11, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 292, 62, 48369, 3419, 4083, 1102, 9246, 9783, 198, 50270, 1309, 30240, 796, 2872, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 39434, 79, 43691, 8, 1391, 198, 50274, 3373, 3461, 453, 11, 428, 318, 18022, 62, 24874, 62, 268, 8189, 39434, 79, 43691, 11, 1222, 944, 13, 12685, 12342, 8, 198, 50274, 3373, 887, 356, 1244, 423, 5495, 257, 40364, 6626, 543, 561, 2948, 4017, 3212, 13, 198, 50274, 3373, 357, 31722, 1744, 287, 262, 4931, 286, 21354, 40364, 30778, 8, 198, 50274, 3373, 1406, 10385, 284, 41002, 12, 23, 290, 466, 40364, 26021, 13, 198, 50274, 3373, 412, 13, 70, 13, 351, 537, 3064, 74, 62, 8692, 366, 220, 220, 2474, 3011, 6626, 284, 366, 366, 1343, 366, 5145, 1600, 198, 50274, 3373, 475, 18022, 62, 24874, 62, 268, 8189, 7203, 220, 220, 2474, 8, 14512, 18022, 62, 24874, 62, 268, 8189, 7203, 366, 8, 198, 50274, 6762, 7, 82, 8, 5218, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 82, 828, 628, 50274, 3373, 5429, 1146, 11, 1771, 393, 407, 428, 3211, 318, 3376, 8338, 319, 1771, 612, 198, 50274, 3373, 561, 307, 257, 40364, 6626, 878, 262, 41002, 12, 23, 40122, 341, 966, 13, 198, 50274, 3373, 18578, 21404, 1576, 326, 645, 530, 481, 1683, 4003, 357, 8499, 477, 11, 661, 1422, 470, 198, 50274, 3373, 4003, 477, 262, 1263, 10421, 287, 262, 2180, 21354, 11241, 7822, 8, 198, 50274, 41512, 28264, 8, 5218, 18022, 62, 24874, 62, 268, 8189, 39434, 79, 43691, 11, 1222, 944, 13, 12685, 12342, 828, 198, 50274, 3373, 13742, 588, 262, 1708, 318, 19827, 475, 11491, 25, 198, 50274, 3373, 41512, 7, 68, 8, 5218, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 13271, 8635, 1391, 198, 50274, 3373, 50259, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 62, 403, 26752, 39434, 79, 43691, 58, 492, 68, 13, 12102, 62, 929, 62, 1462, 3419, 12962, 198, 50274, 3373, 1782, 828, 198, 50270, 18083, 198, 50270, 1309, 4517, 33756, 796, 38692, 3712, 3605, 9783, 198, 50270, 1309, 4517, 33756, 62, 11925, 796, 657, 26, 198, 50270, 329, 11241, 287, 30240, 1391, 198, 50274, 33756, 13, 14689, 7, 30001, 1776, 198, 50274, 33756, 62, 11925, 15853, 2116, 13, 12501, 12342, 58, 5, 30001, 4083, 11925, 9783, 198, 50274, 611, 33756, 62, 11925, 18189, 21354, 62, 33661, 13, 11925, 3419, 1391, 198, 50278, 2270, 26, 198, 50274, 1782, 198, 50270, 1782, 198, 50270, 1224, 45240, 13, 28463, 7, 41068, 1776, 198, 50270, 966, 15853, 352, 26, 198, 50266, 1782, 198, 50262, 1782, 628, 50262, 3373, 770, 318, 635, 407, 15836, 13, 2893, 356, 4143, 7048, 326, 40364, 30778, 389, 8245, 11, 198, 50262, 3373, 12716, 11, 484, 389, 407, 13, 1320, 318, 11, 611, 4375, 9881, 547, 284, 787, 257, 6626, 1656, 287, 198, 50262, 3373, 21354, 62, 33661, 11, 428, 714, 787, 16326, 1744, 543, 674, 9156, 561, 4306, 892, 198, 50262, 3373, 561, 307, 23791, 13, 198, 50262, 3373, 1114, 1672, 11, 351, 308, 457, 17, 11, 262, 779, 286, 3467, 82, 33747, 12248, 59, 50, 8, 1724, 326, 37082, 77, 59, 77, 1, 714, 198, 50262, 3373, 1205, 257, 6626, 11, 304, 13, 70, 13, 37082, 77, 59, 77, 15, 1, 30778, 656, 37082, 77, 1, 10, 1, 59, 77, 1, 10, 1, 15, 1600, 1642, 37082, 77, 1, 257, 1744, 11241, 13, 198, 50262, 3373, 3423, 318, 257, 2068, 290, 11841, 4259, 25, 198, 50262, 3373, 770, 2125, 470, 826, 611, 356, 1683, 4781, 3467, 82, 33747, 12248, 59, 50, 8, 198, 50262, 611, 21354, 62, 33661, 13, 11925, 3419, 1875, 352, 1391, 198, 50266, 1309, 938, 62, 12501, 9043, 796, 275, 2536, 3712, 12501, 1098, 62, 12957, 62, 40477, 23, 7, 403, 31284, 62, 33661, 13, 292, 62, 48369, 35430, 198, 50266, 611, 21354, 62, 33661, 13, 11925, 3419, 532, 938, 62, 12501, 9043, 13, 16, 1875, 657, 198, 50270, 11405, 938, 62, 12501, 9043, 13, 15, 13, 8899, 62, 273, 7, 9562, 11, 930, 66, 91, 269, 13, 271, 62, 1929, 2737, 10223, 28955, 198, 50266, 1391, 198, 50270, 1309, 4517, 302, 12685, 9043, 796, 18022, 62, 24874, 62, 268, 8189, 7, 198, 50274, 1222, 403, 31284, 62, 33661, 58, 492, 403, 31284, 62, 33661, 13, 11925, 3419, 532, 938, 62, 12501, 9043, 13, 16, 4357, 198, 50274, 1222, 944, 13, 12685, 12342, 11, 198, 50270, 5619, 198, 50270, 302, 12685, 9043, 13, 2302, 437, 7, 26327, 62, 24874, 62, 268, 8189, 7, 198, 50274, 1222, 403, 31284, 62, 33661, 58, 403, 31284, 62, 33661, 13, 11925, 3419, 532, 938, 62, 12501, 9043, 13, 16, 492, 4357, 198, 50274, 1222, 944, 13, 12685, 12342, 11, 198, 50270, 29226, 198, 50270, 1224, 45240, 13, 28463, 7, 1361, 40976, 1776, 198, 50266, 1782, 198, 50262, 1782, 628, 50262, 357, 83, 482, 641, 11, 1224, 45240, 8, 198, 50258, 1782, 198, 92, 198, 198, 2, 58, 9078, 24396, 82, 60, 198, 23928, 7231, 33, 11401, 1391, 198, 50258, 1303, 58, 3605, 60, 198, 50258, 24714, 649, 7, 198, 50262, 2207, 12342, 25, 21059, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 22330, 198, 50262, 2041, 62, 83, 482, 641, 62, 12685, 12342, 25, 21059, 13912, 27, 10100, 11, 514, 1096, 22330, 198, 50262, 3912, 25, 1222, 2536, 11, 198, 50258, 1267, 4613, 9485, 23004, 27, 24704, 29, 1391, 198, 50262, 1309, 40364, 796, 797, 25636, 3712, 3605, 7, 33279, 8, 198, 50266, 764, 8899, 62, 8056, 7, 91, 68, 91, 9485, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 11395, 12331, 11, 4808, 33994, 68, 13, 1462, 62, 8841, 3419, 4008, 30, 26, 628, 50262, 1309, 2041, 62, 260, 25636, 796, 1391, 198, 50266, 1309, 4808, 42632, 796, 2041, 62, 83, 482, 641, 62, 12685, 12342, 198, 50270, 764, 13083, 3419, 198, 50270, 764, 8899, 7, 91, 82, 91, 14996, 62, 260, 25636, 3712, 41915, 7, 82, 4008, 198, 50270, 764, 33327, 3712, 27, 53, 721, 27, 62, 4211, 9783, 198, 50266, 797, 25636, 3712, 3605, 39434, 62, 42632, 13, 22179, 7203, 91, 48774, 198, 50270, 764, 8899, 62, 8056, 7, 91, 68, 91, 9485, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 11395, 12331, 11, 4808, 33994, 68, 13, 1462, 62, 8841, 3419, 4008, 30, 198, 50262, 18083, 628, 50262, 1309, 875, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 796, 198, 50266, 2207, 12342, 13, 2676, 22446, 8899, 7, 91, 7, 74, 11, 410, 14726, 20789, 85, 11, 479, 13, 21018, 28955, 737, 33327, 9783, 628, 50262, 6818, 0, 7, 12685, 12342, 13, 11925, 3419, 6624, 875, 12342, 13, 11925, 35430, 628, 50262, 1309, 2041, 62, 83, 482, 641, 62, 12501, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 796, 2041, 62, 83, 482, 641, 62, 12685, 12342, 198, 50266, 764, 2676, 3419, 198, 50266, 764, 8899, 7, 91, 7, 74, 11, 410, 14726, 20789, 85, 11, 479, 13, 292, 62, 33661, 22446, 1462, 62, 35138, 3419, 4008, 198, 50266, 764, 33327, 9783, 628, 50262, 3373, 30698, 780, 314, 836, 470, 760, 703, 284, 1560, 17103, 314, 1101, 407, 1016, 284, 1487, 262, 3975, 198, 50262, 1309, 4517, 23243, 62, 30001, 62, 33661, 25, 38692, 27, 53, 721, 27, 84, 23, 4211, 796, 2207, 12342, 13, 13083, 22446, 565, 12004, 22446, 33327, 9783, 198, 50262, 23243, 62, 30001, 62, 33661, 13, 30619, 9783, 628, 50262, 6762, 7, 14055, 33, 11401, 1391, 198, 50266, 2207, 12342, 11, 198, 50266, 2041, 62, 83, 482, 641, 62, 12685, 12342, 11, 198, 50266, 875, 12342, 11, 198, 50266, 2041, 62, 83, 482, 641, 62, 12501, 12342, 11, 198, 50266, 40364, 62, 83, 7278, 25, 357, 15, 492, 22921, 62, 41359, 62, 4221, 15675, 50, 737, 8899, 7, 91, 62, 91, 40364, 13, 21018, 3419, 737, 33327, 22784, 198, 50266, 2041, 62, 260, 25636, 62, 83, 7278, 25, 357, 15, 492, 22921, 62, 41359, 62, 4221, 15675, 50, 8, 198, 50270, 764, 8899, 7, 91, 62, 91, 2041, 62, 260, 25636, 13, 21018, 28955, 198, 50270, 764, 33327, 22784, 198, 50266, 23243, 62, 30001, 62, 33661, 11, 198, 50262, 32092, 198, 50258, 1782, 628, 50258, 3373, 36658, 18604, 198, 50258, 3373, 14711, 7656, 198, 50258, 3373, 36658, 18604, 628, 50258, 24714, 37773, 62, 35947, 39434, 944, 11, 12972, 25, 11361, 11, 2420, 25, 1222, 2536, 8, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 50262, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 5239, 4008, 198, 50258, 1782, 628, 50258, 24714, 37773, 39434, 944, 11, 12972, 25, 11361, 11, 2420, 25, 1222, 2536, 11, 3142, 62, 20887, 25, 21059, 7248, 27, 5, 2536, 43734, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 50262, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 268, 8189, 62, 30191, 7, 5239, 11, 1222, 40845, 62, 20887, 737, 15, 8, 198, 50258, 1782, 628, 50258, 24714, 4808, 268, 8189, 62, 33661, 39434, 944, 11, 12972, 25, 11361, 11, 9881, 25, 1222, 58, 84, 23, 12962, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 50262, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 1391, 198, 50266, 2872, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 7, 33661, 8, 1391, 198, 50270, 6762, 7, 5239, 8, 5218, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 5239, 828, 198, 50270, 41512, 7, 68, 8, 5218, 1391, 198, 50274, 1309, 2420, 796, 21596, 1391, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 62, 403, 26752, 39434, 33661, 58, 492, 68, 13, 12102, 62, 929, 62, 1462, 3419, 12962, 18083, 198, 50274, 1309, 357, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 2116, 13557, 268, 8189, 62, 30191, 7, 5239, 11, 1222, 26257, 7248, 3712, 3605, 35430, 198, 50274, 1309, 357, 21973, 16326, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 198, 50278, 2116, 13557, 24988, 589, 62, 12957, 62, 12239, 62, 30001, 62, 11925, 7, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 198, 50274, 611, 5145, 83, 482, 641, 13, 271, 62, 28920, 3419, 11405, 938, 62, 12239, 62, 30001, 62, 11925, 1875, 657, 1391, 198, 50278, 3373, 406, 404, 572, 262, 16326, 422, 262, 938, 3704, 290, 1057, 347, 11401, 319, 262, 5637, 9881, 198, 50278, 3373, 32499, 5183, 21404, 11, 475, 428, 743, 407, 307, 3376, 611, 356, 1549, 423, 550, 257, 40364, 198, 50278, 3373, 6626, 1022, 262, 4938, 41002, 12, 23, 290, 262, 12515, 9881, 11, 543, 318, 1521, 428, 198, 50278, 3373, 2446, 318, 2839, 198, 50278, 1309, 4517, 21354, 62, 33661, 796, 198, 50271, 50266, 2116, 13557, 12501, 1098, 62, 30191, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 492, 36563, 198, 50278, 21354, 62, 33661, 13, 2302, 437, 62, 6738, 62, 48369, 39434, 33661, 58, 68, 13, 12102, 62, 929, 62, 1462, 3419, 492, 36563, 628, 50278, 16326, 13, 2213, 19524, 378, 7, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 198, 50278, 16326, 13, 2302, 437, 7, 26327, 62, 24874, 62, 268, 8189, 39434, 403, 31284, 62, 33661, 11, 1222, 944, 13, 12685, 12342, 18125, 198, 50274, 1782, 198, 50274, 16326, 198, 50270, 1782, 198, 50266, 1782, 198, 50262, 32092, 198, 50258, 1782, 628, 50258, 24714, 37773, 62, 4480, 62, 403, 31284, 7, 198, 50262, 1222, 944, 11, 198, 50262, 12972, 25, 11361, 11, 198, 50262, 2420, 25, 1222, 2536, 11, 198, 50262, 3142, 62, 20887, 25, 21059, 7248, 27, 5, 2536, 22330, 198, 50258, 1267, 4613, 9485, 27, 20519, 51, 29291, 29, 1391, 198, 50262, 1309, 357, 83, 482, 641, 11, 1224, 45240, 8, 796, 198, 50266, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 268, 8189, 62, 403, 31284, 62, 30191, 7, 5239, 11, 1222, 40845, 62, 20887, 18125, 198, 50262, 1309, 12972, 62, 785, 37069, 507, 796, 198, 50266, 9485, 8053, 3712, 3605, 7, 9078, 11, 1224, 45240, 13, 2676, 22446, 8899, 7, 91, 41068, 91, 9485, 8053, 3712, 3605, 7, 9078, 11, 1222, 41068, 58, 492, 60, 4008, 1776, 198, 50262, 357, 83, 482, 641, 11, 12972, 62, 785, 37069, 507, 737, 20424, 62, 9078, 7, 9078, 8, 198, 50258, 1782, 628, 50258, 24714, 37773, 62, 29762, 62, 30001, 39434, 944, 11, 3704, 25, 1222, 58, 84, 23, 12962, 4613, 9485, 23004, 27, 385, 1096, 29, 1391, 198, 50262, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 737, 22163, 798, 3419, 1391, 198, 50266, 1441, 6762, 7, 30001, 1776, 198, 50262, 1782, 198, 50262, 611, 1309, 6762, 7, 12239, 62, 2536, 8, 796, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 7, 12239, 8, 1391, 198, 50266, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 20887, 62, 83, 482, 641, 62, 12685, 12342, 13, 1136, 7, 12239, 62, 2536, 737, 22163, 798, 3419, 1391, 198, 50270, 1441, 6762, 7, 30001, 1776, 198, 50266, 1782, 198, 50262, 1782, 198, 50262, 41512, 7, 20519, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 9218, 12331, 11, 4808, 33994, 12239, 13, 1462, 62, 11990, 3419, 4008, 198, 50258, 1782, 628, 50258, 24714, 37773, 62, 29762, 62, 12239, 39434, 944, 11, 3704, 25, 1222, 58, 84, 23, 12962, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 50262, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 8, 1391, 198, 50266, 1441, 43030, 0, 58, 9, 30001, 11208, 198, 50262, 1782, 198, 50262, 18022, 62, 24874, 62, 268, 8189, 7, 12239, 11, 1222, 944, 13, 12685, 12342, 8, 198, 50258, 1782, 628, 50258, 3373, 36658, 18604, 198, 50258, 3373, 4280, 7656, 198, 50258, 3373, 36658, 18604, 628, 50258, 24714, 36899, 62, 33661, 39434, 944, 11, 12972, 25, 11361, 11, 16326, 25, 38692, 27, 385, 1096, 43734, 4613, 9485, 27, 20519, 45992, 29, 1391, 198, 50262, 1309, 9881, 796, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 12501, 1098, 62, 30191, 39434, 83, 482, 641, 18125, 198, 50262, 9485, 45992, 3712, 3605, 7, 9078, 11, 1222, 33661, 737, 20424, 3419, 198, 50258, 1782, 628, 50258, 24714, 36899, 62, 29762, 62, 30001, 62, 33661, 39434, 944, 11, 12972, 25, 11361, 11, 11241, 25, 514, 1096, 8, 4613, 9485, 23004, 27, 20519, 27, 20519, 45992, 4211, 1391, 198, 50262, 611, 1309, 2773, 7, 33661, 8, 796, 2116, 13, 12501, 12342, 13, 1136, 39434, 30001, 8, 1391, 198, 50266, 1441, 6762, 7, 20519, 45992, 3712, 3605, 7, 9078, 11, 9881, 737, 20424, 35430, 198, 50262, 1782, 198, 50262, 611, 1309, 2773, 7, 33661, 8, 796, 2116, 13, 20887, 62, 83, 482, 641, 62, 12501, 12342, 13, 1136, 39434, 30001, 8, 1391, 198, 50266, 1441, 6762, 7, 20519, 45992, 3712, 3605, 7, 9078, 11, 9881, 737, 20424, 35430, 198, 50262, 1782, 198, 50262, 41512, 7, 20519, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 9218, 12331, 11, 4808, 33994, 30001, 13, 1462, 62, 8841, 3419, 4008, 198, 50258, 1782, 628, 50258, 3373, 36658, 18604, 198, 50258, 3373, 46253, 198, 50258, 3373, 36658, 18604, 628, 50258, 24714, 11241, 62, 26327, 62, 27160, 39434, 944, 11, 12972, 25, 11361, 8, 4613, 38692, 27, 20519, 27, 20519, 45992, 4211, 1391, 198, 50262, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 198, 50266, 764, 2676, 3419, 198, 50266, 764, 8899, 7, 91, 87, 91, 9485, 45992, 3712, 3605, 7, 9078, 11, 2124, 737, 20424, 28955, 198, 50266, 764, 33327, 3419, 198, 50258, 1782, 198, 92, 198, 198, 2, 58, 9078, 21412, 60, 198, 22184, 4808, 83, 1134, 30001, 28264, 9078, 25, 11361, 11, 285, 25, 1222, 20519, 26796, 8, 4613, 9485, 23004, 27, 3419, 29, 1391, 198, 50258, 285, 13, 2860, 62, 4871, 3712, 27, 14055, 33, 11401, 29, 3419, 30, 26, 198, 50258, 6762, 7, 28955, 198, 92, 198, 198, 2, 58, 37581, 7, 9288, 15437, 198, 4666, 5254, 1391, 198, 50258, 779, 17000, 66, 62, 17831, 3712, 37, 87, 26257, 13912, 355, 21059, 13912, 26, 628, 50258, 779, 27021, 3712, 26327, 62, 24874, 62, 35312, 26, 628, 50258, 1303, 58, 9288, 60, 198, 50258, 24714, 845, 62, 36439, 62, 9288, 3419, 1391, 198, 50262, 1309, 4517, 9803, 796, 21059, 13912, 3712, 12286, 9783, 198, 50262, 9803, 13, 28463, 7, 65, 1, 397, 1911, 1462, 62, 35138, 22784, 352, 1776, 198, 50262, 9803, 13, 28463, 7, 65, 1, 10210, 1911, 1462, 62, 35138, 22784, 362, 1776, 628, 50262, 1309, 581, 796, 18022, 62, 24874, 62, 35312, 7, 65, 1, 397, 10210, 1600, 1222, 81, 2283, 1776, 198, 50262, 6818, 62, 27363, 0, 7, 411, 11, 43030, 0, 58, 65, 1, 397, 1600, 275, 1, 10210, 8973, 1776, 198, 50258, 1782, 198, 92, 198] \ No newline at end of file diff --git a/test/Microsoft.ML.Tokenizers.Tests/Data/tokens_r50k_base.json b/test/Microsoft.ML.Tokenizers.Tests/Data/tokens_r50k_base.json new file mode 100644 index 0000000000..e389642d70 --- /dev/null +++ b/test/Microsoft.ML.Tokenizers.Tests/Data/tokens_r50k_base.json @@ -0,0 +1 @@ +[1003, 770, 2198, 318, 649, 290, 2331, 46542, 357, 39363, 351, 9485, 46, 18, 10375, 8, 198, 2, 0, 58, 12154, 7, 565, 41214, 3712, 2865, 808, 62, 67, 567, 69, 62, 5420, 15437, 198, 198, 1904, 14367, 3712, 4033, 26448, 3712, 26257, 7248, 26, 198, 1904, 14367, 3712, 16663, 26, 198, 198, 1904, 14996, 62, 260, 25636, 3712, 3041, 25636, 26, 198, 1904, 279, 8226, 18, 3712, 1069, 11755, 26, 198, 1904, 279, 8226, 18, 3712, 79, 2411, 2507, 3712, 9, 26, 198, 1904, 279, 8226, 18, 3712, 19199, 3712, 90, 20519, 45992, 11, 9485, 8053, 11, 9485, 51, 29291, 19629, 198, 1904, 279, 8226, 18, 3712, 20519, 23004, 26, 198, 1904, 17000, 66, 62, 17831, 3712, 37, 87, 26257, 13912, 355, 21059, 13912, 26, 198, 198, 22184, 4808, 26327, 62, 24874, 62, 647, 469, 27, 51, 33994, 198, 220, 220, 220, 3704, 25, 1222, 58, 84, 23, 4357, 198, 220, 220, 220, 9803, 25, 1222, 26257, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 22330, 198, 220, 220, 220, 277, 25, 4114, 37481, 7, 19282, 3712, 2840, 3712, 17257, 27, 385, 1096, 43734, 4613, 309, 11, 198, 8, 4613, 38692, 27, 51, 29, 1391, 198, 220, 220, 220, 3373, 770, 318, 257, 15879, 286, 357, 9688, 11, 4279, 737, 198, 220, 220, 220, 3373, 383, 4279, 318, 286, 262, 18022, 5166, 3599, 379, 2292, 923, 13, 198, 220, 220, 220, 3373, 383, 4279, 286, 262, 938, 2378, 287, 262, 15879, 318, 407, 257, 4938, 1988, 13, 198, 220, 220, 220, 1309, 4517, 3354, 25, 38692, 27, 7, 385, 1096, 11, 514, 1096, 8, 29, 796, 357, 15, 492, 12239, 13, 11925, 3419, 1343, 352, 737, 8899, 7, 91, 72, 91, 357, 72, 11, 514, 1096, 3712, 22921, 29720, 33327, 9783, 628, 220, 220, 220, 1309, 651, 62, 43027, 796, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1303, 58, 45145, 7, 33770, 15437, 198, 220, 220, 220, 220, 220, 220, 220, 930, 42632, 25, 1222, 53, 721, 27, 7, 385, 1096, 11, 514, 1096, 8, 22330, 923, 62, 312, 87, 25, 514, 1096, 11, 14267, 25, 514, 1096, 91, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 357, 9688, 62, 312, 87, 1343, 14267, 1343, 362, 8, 1279, 3354, 13, 11925, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 9803, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 1136, 39434, 12239, 58, 42632, 58, 9688, 62, 312, 87, 4083, 15, 492, 42632, 58, 9688, 62, 312, 87, 1343, 14267, 1343, 362, 4083, 15, 12962, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 22163, 798, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 2073, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 6045, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 18083, 628, 220, 220, 220, 3373, 775, 804, 510, 262, 9803, 1752, 287, 262, 3726, 290, 11629, 9404, 4296, 198, 220, 220, 220, 3373, 606, 1141, 1123, 20121, 11, 543, 12850, 262, 1271, 286, 4279, 804, 4739, 13, 198, 220, 220, 220, 329, 1312, 287, 657, 492, 42632, 13, 11925, 3419, 532, 362, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 2872, 651, 62, 43027, 39434, 42632, 11, 1312, 11, 657, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2773, 7, 43027, 8, 5218, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 514, 1096, 3712, 22921, 318, 257, 1908, 20538, 1988, 290, 2314, 307, 257, 4938, 4279, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 14257, 62, 30493, 0, 7, 43027, 14512, 514, 1096, 3712, 22921, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3354, 58, 72, 4083, 16, 796, 4279, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 6045, 5218, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2555, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 18083, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 3373, 1002, 345, 423, 299, 3354, 290, 285, 4017, 3212, 11, 428, 857, 440, 7, 10295, 8, 670, 13, 198, 220, 220, 220, 3373, 775, 714, 466, 1223, 351, 257, 24575, 290, 466, 440, 7, 76, 2604, 299, 8, 670, 13, 198, 220, 220, 220, 3373, 632, 318, 1593, 284, 2074, 326, 299, 318, 1690, 1402, 38155, 3064, 828, 290, 355, 884, 198, 220, 220, 220, 3373, 262, 12940, 12, 17946, 1483, 4034, 31405, 262, 8385, 9383, 13357, 21838, 1460, 198, 220, 220, 220, 3373, 286, 262, 4600, 42632, 63, 15879, 1366, 4645, 2029, 13, 628, 220, 220, 220, 3373, 5740, 326, 356, 12234, 9881, 11, 407, 11241, 14729, 13, 1081, 890, 355, 356, 4512, 347, 11401, 262, 835, 356, 198, 220, 220, 220, 3373, 3058, 466, 11, 428, 318, 7548, 13, 1052, 2562, 835, 284, 2270, 428, 561, 307, 284, 875, 43846, 198, 220, 220, 220, 3373, 20121, 8475, 422, 11241, 6376, 393, 284, 2948, 2176, 11241, 4017, 3212, 13, 198, 220, 220, 220, 9052, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 611, 3354, 13, 11925, 3419, 6624, 352, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2270, 26, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 3373, 514, 1096, 3712, 22921, 318, 257, 1908, 20538, 4279, 1988, 5086, 514, 284, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 1011, 262, 949, 517, 2952, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 949, 62, 43027, 25, 357, 385, 1096, 11, 514, 1096, 8, 796, 357, 385, 1096, 3712, 22921, 11, 657, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 329, 357, 72, 11, 1222, 28264, 11, 4279, 4008, 287, 3354, 58, 492, 42632, 13, 11925, 3419, 532, 352, 4083, 2676, 22446, 268, 6975, 378, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 4279, 1279, 949, 62, 43027, 13, 15, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 949, 62, 43027, 796, 357, 43027, 11, 1312, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 611, 949, 62, 43027, 13, 15, 14512, 514, 1096, 3712, 22921, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 1312, 796, 949, 62, 43027, 13, 16, 26, 628, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 24550, 25, 775, 389, 546, 284, 4781, 3354, 58, 72, 1343, 352, 4083, 775, 466, 407, 466, 340, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 1865, 780, 612, 389, 12940, 12, 17946, 1483, 4034, 284, 19698, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 3354, 58, 72, 60, 290, 3354, 58, 72, 12, 16, 60, 878, 10829, 11, 543, 714, 5636, 1077, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 262, 12940, 13, 6660, 11, 356, 4296, 262, 4279, 17952, 416, 31017, 625, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 3354, 58, 72, 1343, 352, 4357, 416, 39744, 4600, 1136, 62, 43027, 0, 63, 351, 4600, 48267, 796, 352, 44646, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3354, 58, 72, 4083, 16, 796, 651, 62, 43027, 39434, 42632, 11, 1312, 11, 352, 737, 403, 37150, 62, 273, 7, 385, 1096, 3712, 22921, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 1312, 1875, 657, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3354, 58, 72, 532, 352, 4083, 16, 796, 651, 62, 43027, 39434, 42632, 11, 1312, 532, 352, 11, 352, 737, 403, 37150, 62, 273, 7, 385, 1096, 3712, 22921, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3354, 13, 28956, 7, 72, 1343, 352, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 2073, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2270, 26, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 1782, 198, 220, 220, 220, 1309, 4517, 503, 25, 38692, 27, 51, 29, 796, 38692, 3712, 4480, 62, 42404, 7, 42632, 13, 11925, 3419, 532, 352, 1776, 198, 220, 220, 220, 329, 1312, 287, 657, 492, 42632, 13, 11925, 3419, 532, 352, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 503, 13, 14689, 7, 69, 7, 42632, 58, 72, 4083, 15, 492, 42632, 58, 72, 1343, 352, 4083, 15, 18125, 198, 220, 220, 220, 1782, 198, 220, 220, 220, 503, 198, 92, 198, 198, 12984, 24714, 18022, 62, 24874, 62, 268, 8189, 7, 12239, 25, 1222, 58, 84, 23, 4357, 9803, 25, 1222, 26257, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 43734, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 220, 220, 220, 611, 3704, 13, 11925, 3419, 6624, 352, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1441, 43030, 0, 58, 81, 2283, 58, 12239, 60, 11208, 198, 220, 220, 220, 1782, 198, 220, 220, 220, 4808, 26327, 62, 24874, 62, 647, 469, 7, 12239, 11, 9803, 11, 930, 79, 91, 9803, 58, 5, 12239, 58, 79, 13, 9688, 492, 79, 13, 437, 11907, 8, 198, 92, 198, 198, 12984, 24714, 18022, 62, 24874, 62, 35312, 27, 6, 64, 33994, 12239, 25, 1222, 6, 64, 685, 84, 23, 4357, 9803, 25, 1222, 26257, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 43734, 4613, 38692, 27, 5, 6, 64, 685, 84, 23, 60, 29, 1391, 198, 220, 220, 220, 611, 3704, 13, 11925, 3419, 6624, 352, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1441, 43030, 0, 58, 12239, 11208, 198, 220, 220, 220, 1782, 198, 220, 220, 220, 4808, 26327, 62, 24874, 62, 647, 469, 7, 12239, 11, 9803, 11, 930, 79, 91, 1222, 12239, 58, 79, 13, 9688, 492, 79, 13, 437, 12962, 198, 92, 198, 198, 1003, 26386, 2854, 4710, 25, 198, 1003, 198, 1003, 797, 25636, 198, 1003, 29335, 198, 1003, 4042, 286, 262, 640, 318, 3377, 287, 40364, 13, 383, 16638, 835, 284, 2866, 428, 510, 318, 416, 1262, 1342, 14996, 198, 1003, 40364, 3033, 13, 1114, 4554, 11, 1262, 257, 40364, 21136, 12, 540, 416, 4600, 260, 25636, 63, 27021, 318, 513, 87, 5443, 621, 198, 1003, 262, 6678, 40364, 356, 779, 13, 198, 1003, 198, 1003, 2102, 11, 1813, 326, 356, 821, 1262, 257, 40364, 21136, 12, 540, 416, 4600, 260, 25636, 47671, 612, 2125, 470, 881, 3580, 198, 1003, 1022, 1262, 262, 4600, 260, 25636, 63, 27021, 290, 1262, 262, 4600, 69, 3883, 62, 260, 25636, 63, 27021, 13, 198, 1003, 198, 1003, 1318, 318, 281, 1593, 10375, 1022, 4704, 278, 11, 4600, 260, 25636, 63, 290, 4600, 69, 3883, 62, 260, 25636, 44646, 198, 1003, 1649, 1262, 4600, 69, 3883, 62, 260, 25636, 47671, 356, 2277, 4600, 260, 25636, 13, 19796, 62, 265, 44646, 632, 4962, 503, 326, 428, 5640, 22627, 319, 198, 1003, 617, 4517, 540, 12692, 2272, 2641, 286, 4600, 260, 25636, 44646, 770, 5543, 12847, 2854, 13, 1649, 1262, 8631, 198, 1003, 1468, 4600, 260, 25636, 47671, 356, 836, 470, 2277, 428, 11, 780, 4600, 19796, 62, 2676, 63, 468, 257, 1180, 2438, 3108, 13, 198, 1003, 19809, 25, 3740, 1378, 12567, 13, 785, 14, 11469, 12, 17204, 14, 260, 25636, 14, 2436, 672, 14, 9866, 14, 18973, 13775, 10725, 5222, 13, 9132, 198, 1003, 21836, 11, 262, 835, 356, 651, 1088, 428, 318, 351, 1719, 257, 357, 29471, 8, 4704, 1957, 17271, 286, 262, 40364, 329, 198, 1003, 1123, 4704, 13, 198, 1003, 198, 1003, 14122, 278, 198, 1003, 796, 2559, 198, 1003, 314, 3088, 1262, 4600, 2433, 261, 44646, 632, 2492, 470, 1107, 5443, 621, 1262, 11361, 14390, 290, 13011, 262, 402, 4146, 13, 198, 1003, 1406, 24829, 4600, 2433, 261, 63, 0, 3914, 4704, 954, 3503, 307, 287, 1630, 286, 674, 11361, 2985, 13, 198, 1003, 198, 1003, 327, 8103, 198, 1003, 29335, 855, 198, 1003, 383, 4941, 11241, 5847, 468, 281, 300, 622, 12940, 625, 262, 7548, 286, 4600, 26327, 62, 24874, 62, 268, 8189, 44646, 198, 1003, 19486, 11, 356, 550, 530, 1165, 0, 9170, 340, 11, 356, 547, 691, 29627, 5443, 621, 11361, 13, 198, 1003, 314, 973, 281, 33212, 25392, 284, 1805, 262, 12940, 13, 770, 1422, 470, 1283, 284, 5938, 2060, 40945, 2854, 198, 1003, 33752, 11, 475, 340, 750, 2689, 5021, 12, 16663, 276, 2854, 13, 34097, 306, 11, 340, 3947, 284, 2689, 198, 1003, 5021, 12, 16663, 276, 2854, 772, 618, 314, 691, 550, 7183, 357, 11261, 3077, 314, 32621, 1223, 510, 29865, 198, 1003, 21836, 11, 314, 19169, 326, 356, 714, 651, 5755, 286, 262, 12940, 11, 611, 356, 2190, 262, 900, 286, 16326, 355, 257, 12940, 0, 198, 1003, 2312, 389, 3446, 262, 900, 393, 4017, 3212, 326, 389, 1884, 284, 307, 3024, 13, 843, 783, 356, 836, 470, 423, 284, 892, 198, 1003, 546, 11087, 4517, 1799, 11, 4088, 779, 11, 393, 45973, 13, 198, 1003, 198, 1003, 367, 2140, 198, 1003, 29335, 855, 198, 1003, 775, 779, 376, 87, 26257, 13912, 2427, 286, 262, 3210, 21059, 13912, 13, 770, 318, 3863, 588, 257, 642, 12, 940, 4, 1592, 30, 198, 1003, 383, 1459, 7822, 5645, 510, 1804, 257, 1256, 286, 49544, 286, 9881, 13, 554, 4583, 11, 428, 714, 307, 925, 198, 1003, 284, 307, 49544, 286, 734, 12, 28047, 2374, 286, 493, 82, 11, 543, 3073, 588, 340, 743, 635, 307, 257, 3155, 1411, 5443, 13, 198, 198, 1904, 14367, 3712, 22510, 3712, 15419, 28667, 52, 2414, 26, 198, 12984, 2878, 33482, 16818, 7390, 7, 15419, 28667, 52, 2414, 1776, 198, 198, 22184, 12234, 62, 14421, 62, 16663, 3419, 4613, 514, 1096, 1391, 198, 220, 220, 220, 3373, 632, 338, 4577, 284, 779, 21596, 621, 284, 779, 37862, 13, 17103, 468, 428, 3621, 334, 2414, 4704, 4686, 3753, 198, 220, 220, 220, 3373, 326, 2499, 1049, 329, 674, 779, 1339, 286, 14928, 31998, 287, 674, 7177, 13, 8989, 11, 198, 220, 220, 220, 3373, 340, 338, 2839, 13, 2102, 11, 612, 389, 691, 523, 867, 2842, 345, 460, 12461, 257, 334, 2414, 11, 523, 655, 21595, 1133, 198, 220, 220, 220, 3373, 3740, 1378, 12567, 13, 785, 14, 11469, 12, 17204, 14, 11469, 14, 37165, 14, 37601, 2670, 198, 220, 220, 220, 1500, 4808, 25, 685, 84, 23, 26, 807, 60, 796, 685, 15, 26, 14367, 3712, 11883, 3712, 7857, 62, 1659, 3712, 27, 19282, 3712, 16663, 3712, 16818, 7390, 29, 3419, 11208, 198, 220, 220, 220, 1500, 4808, 25, 685, 84, 23, 26, 807, 60, 796, 685, 15, 26, 14367, 3712, 11883, 3712, 7857, 62, 1659, 3712, 27, 49233, 16818, 7390, 29, 3419, 11208, 198, 220, 220, 220, 1309, 2124, 796, 21596, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 14367, 3712, 11883, 3712, 7645, 76, 1133, 3712, 27, 19282, 3712, 16663, 3712, 16818, 7390, 11, 33482, 16818, 7390, 33994, 16663, 3712, 14421, 22446, 312, 3419, 737, 15, 198, 220, 220, 220, 18083, 198, 220, 220, 220, 334, 2414, 3712, 6738, 7, 87, 8, 355, 514, 1096, 198, 92, 198, 198, 9979, 25882, 62, 41359, 62, 4221, 15675, 50, 25, 514, 1096, 796, 13108, 26, 198, 2, 58, 9078, 4871, 60, 198, 7249, 7231, 33, 11401, 1391, 198, 220, 220, 220, 2207, 12342, 25, 21059, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 22330, 198, 220, 220, 220, 2041, 62, 83, 482, 641, 62, 12685, 12342, 25, 21059, 13912, 27, 10100, 11, 514, 1096, 22330, 198, 220, 220, 220, 875, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 11, 198, 220, 220, 220, 2041, 62, 83, 482, 641, 62, 12501, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 11, 198, 220, 220, 220, 40364, 62, 83, 7278, 25, 38692, 27, 3041, 25636, 22330, 198, 220, 220, 220, 2041, 62, 260, 25636, 62, 83, 7278, 25, 38692, 27, 3041, 25636, 22330, 198, 220, 220, 220, 23243, 62, 30001, 62, 33661, 25, 38692, 27, 53, 721, 27, 84, 23, 4211, 11, 198, 92, 198, 198, 23928, 7231, 33, 11401, 1391, 198, 220, 220, 220, 24714, 4808, 1136, 62, 28781, 62, 260, 25636, 39434, 944, 8, 4613, 1222, 3041, 25636, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 4091, 2854, 4710, 2029, 329, 644, 428, 318, 546, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 632, 338, 635, 257, 1310, 474, 39556, 11, 3387, 787, 257, 1365, 2196, 286, 340, 0, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 2102, 11, 340, 338, 3621, 326, 428, 1595, 470, 13044, 4088, 284, 1790, 12, 24489, 14390, 198, 220, 220, 220, 220, 220, 220, 220, 1222, 944, 13, 260, 25636, 62, 83, 7278, 58, 17831, 62, 14421, 62, 16663, 3419, 4064, 25882, 62, 41359, 62, 4221, 15675, 50, 60, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 4808, 1136, 62, 28781, 62, 20887, 62, 260, 25636, 39434, 944, 8, 4613, 1222, 3041, 25636, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1222, 944, 13, 20887, 62, 260, 25636, 62, 83, 7278, 58, 17831, 62, 14421, 62, 16663, 3419, 4064, 25882, 62, 41359, 62, 4221, 15675, 50, 60, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 4808, 12501, 1098, 62, 30191, 39434, 944, 11, 16326, 25, 1222, 58, 385, 1096, 12962, 4613, 38692, 27, 84, 23, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 1005, 796, 38692, 3712, 4480, 62, 42404, 7, 83, 482, 641, 13, 11925, 3419, 1635, 362, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 329, 11241, 287, 16326, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 11241, 62, 33661, 796, 2116, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 12501, 12342, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 1136, 7, 30001, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 403, 37150, 62, 273, 62, 17772, 7, 15886, 1222, 944, 13, 20887, 62, 83, 482, 641, 62, 12501, 12342, 58, 30001, 36563, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1005, 13, 2302, 437, 7, 30001, 62, 33661, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1005, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 4808, 268, 8189, 62, 35947, 62, 30191, 39434, 944, 11, 2420, 25, 1222, 2536, 8, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 770, 318, 262, 4755, 286, 262, 21004, 9156, 26, 262, 584, 5499, 287, 994, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 655, 787, 1243, 8253, 47226, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 40364, 796, 2116, 13557, 1136, 62, 28781, 62, 260, 25636, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 1005, 796, 43030, 0, 58, 11208, 198, 220, 220, 220, 220, 220, 220, 220, 329, 2603, 287, 40364, 13, 19796, 62, 2676, 7, 5239, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 3704, 796, 2603, 13, 403, 37150, 22446, 292, 62, 2536, 22446, 292, 62, 33661, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1005, 13, 14689, 46491, 30001, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2555, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1005, 13, 2302, 437, 39434, 26327, 62, 24874, 62, 268, 8189, 7, 12239, 11, 1222, 944, 13, 12685, 12342, 18125, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1005, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 4808, 268, 8189, 62, 30191, 39434, 944, 11, 2420, 25, 1222, 2536, 11, 3142, 62, 20887, 25, 1222, 26257, 7248, 27, 5, 2536, 43734, 4613, 357, 53, 721, 27, 385, 1096, 22330, 514, 1096, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 2041, 62, 260, 25636, 796, 2116, 13557, 1136, 62, 28781, 62, 20887, 62, 260, 25636, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 40364, 796, 2116, 13557, 1136, 62, 28781, 62, 260, 25636, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 1005, 796, 43030, 0, 58, 11208, 628, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 923, 796, 657, 26, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 938, 62, 12239, 62, 30001, 62, 11925, 796, 657, 26, 198, 220, 220, 220, 220, 220, 220, 220, 9052, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 1306, 62, 20887, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 923, 62, 19796, 796, 923, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 9052, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 9938, 262, 1306, 3142, 2041, 11241, 11, 611, 597, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1306, 62, 20887, 796, 2041, 62, 260, 25636, 13, 19796, 62, 6738, 62, 1930, 7, 5239, 11, 923, 62, 19796, 737, 403, 37150, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2872, 1306, 62, 20887, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2773, 7, 76, 8, 5218, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 3142, 62, 20887, 13, 3642, 1299, 39434, 5239, 58, 76, 13, 9688, 3419, 492, 76, 13, 437, 3419, 12962, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2270, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 923, 62, 19796, 796, 285, 13, 9688, 3419, 1343, 352, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 6045, 5218, 2270, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 886, 796, 1306, 62, 20887, 13, 8899, 62, 273, 7, 5239, 13, 11925, 22784, 930, 76, 91, 285, 13, 9688, 35430, 628, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 16805, 11, 994, 356, 467, 11, 8996, 428, 9156, 284, 4808, 268, 8189, 62, 35947, 62, 30191, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 329, 2603, 287, 40364, 13, 19796, 62, 2676, 39434, 5239, 58, 9688, 492, 437, 12962, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 3704, 796, 2603, 13, 403, 37150, 22446, 292, 62, 2536, 22446, 292, 62, 33661, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 938, 62, 12239, 62, 30001, 62, 11925, 796, 352, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1005, 13, 14689, 46491, 30001, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2555, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 16326, 796, 18022, 62, 24874, 62, 268, 8189, 7, 12239, 11, 1222, 944, 13, 12685, 12342, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 938, 62, 12239, 62, 30001, 62, 11925, 796, 16326, 13, 11925, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1005, 13, 2302, 437, 39434, 83, 482, 641, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2872, 1306, 62, 20887, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 843, 994, 356, 4574, 262, 2041, 11241, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2773, 7, 76, 8, 5218, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 3704, 796, 285, 13, 292, 62, 2536, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 11241, 796, 2116, 13, 20887, 62, 83, 482, 641, 62, 12685, 12342, 58, 12239, 11208, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1005, 13, 14689, 7, 30001, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 923, 796, 285, 13, 437, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 938, 62, 12239, 62, 30001, 62, 11925, 796, 657, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 6045, 5218, 2270, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 3373, 938, 62, 12239, 62, 30001, 62, 11925, 318, 703, 867, 16326, 1625, 422, 262, 938, 40364, 6626, 13, 770, 318, 973, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 329, 13213, 21354, 16326, 11, 1201, 345, 460, 470, 20121, 1973, 357, 31284, 8, 40364, 30778, 198, 220, 220, 220, 220, 220, 220, 220, 357, 1186, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 4808, 24988, 589, 62, 12957, 62, 12239, 62, 30001, 62, 11925, 7, 198, 220, 220, 220, 220, 220, 220, 220, 1222, 944, 11, 198, 220, 220, 220, 220, 220, 220, 220, 16326, 25, 38692, 27, 385, 1096, 22330, 198, 220, 220, 220, 220, 220, 220, 220, 4517, 938, 62, 12239, 62, 30001, 62, 11925, 25, 514, 1096, 11, 198, 220, 220, 220, 1267, 4613, 357, 53, 721, 27, 385, 1096, 22330, 514, 1096, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 8989, 11, 262, 7064, 810, 674, 40364, 30778, 460, 307, 21354, 13, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 1114, 262, 4959, 286, 13213, 21354, 16326, 11, 21354, 40364, 26021, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 318, 691, 257, 1917, 611, 257, 6626, 326, 373, 1944, 27934, 11, 1201, 428, 460, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 1085, 284, 35981, 286, 16326, 4306, 1807, 284, 307, 8245, 13, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 537, 3064, 74, 62, 8692, 1838, 674, 1204, 1327, 416, 1390, 262, 3467, 82, 9, 58, 59, 81, 59, 77, 48688, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 3912, 13, 770, 460, 304, 13, 70, 13, 2728, 37082, 77, 1, 1343, 366, 366, 284, 1716, 37082, 77, 3467, 77, 1911, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 3423, 318, 257, 2068, 290, 11841, 4259, 25, 198, 220, 220, 220, 220, 220, 220, 220, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 11241, 62, 271, 62, 439, 62, 13200, 796, 930, 30001, 91, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2116, 13, 12501, 12342, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 1136, 7, 30001, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 8899, 7, 91, 30001, 62, 33661, 91, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 11241, 62, 33661, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 2676, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 18218, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 439, 7, 91, 5, 65, 91, 685, 65, 6, 46083, 275, 6, 59, 77, 3256, 275, 6, 59, 83, 6, 4083, 3642, 1299, 39434, 65, 4008, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 32092, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 403, 37150, 62, 273, 7, 9562, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 18083, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 938, 62, 12239, 62, 30001, 62, 11925, 1875, 657, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 11405, 11241, 62, 271, 62, 439, 62, 13200, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 12962, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 981, 357, 12957, 62, 12239, 62, 30001, 62, 11925, 1279, 16326, 13, 11925, 28955, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 11405, 11241, 62, 271, 62, 439, 62, 13200, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 532, 352, 12962, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 938, 62, 12239, 62, 30001, 62, 11925, 15853, 352, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 14257, 62, 30493, 0, 7, 12957, 62, 12239, 62, 30001, 62, 11925, 19841, 16326, 13, 11925, 35430, 628, 220, 220, 220, 220, 220, 220, 220, 357, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 4808, 268, 8189, 62, 403, 31284, 62, 30191, 7, 198, 220, 220, 220, 220, 220, 220, 220, 1222, 944, 11, 198, 220, 220, 220, 220, 220, 220, 220, 2420, 25, 1222, 2536, 11, 198, 220, 220, 220, 220, 220, 220, 220, 3142, 62, 20887, 25, 1222, 26257, 7248, 27, 5, 2536, 22330, 198, 220, 220, 220, 1267, 4613, 357, 53, 721, 27, 385, 1096, 22330, 21059, 7248, 27, 53, 721, 27, 385, 1096, 4211, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 357, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 2116, 13557, 268, 8189, 62, 30191, 7, 5239, 11, 3142, 62, 20887, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 611, 938, 62, 12239, 62, 30001, 62, 11925, 6624, 657, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 1002, 938, 62, 12239, 62, 30001, 62, 11925, 318, 6632, 11, 262, 938, 11241, 373, 257, 2041, 11241, 290, 356, 423, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 645, 21354, 9881, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1441, 357, 83, 482, 641, 11, 21059, 7248, 3712, 3605, 35430, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 357, 21973, 16326, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2116, 13557, 24988, 589, 62, 12957, 62, 12239, 62, 30001, 62, 11925, 7, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 628, 220, 220, 220, 220, 220, 220, 220, 1309, 21354, 62, 33661, 796, 2116, 13557, 12501, 1098, 62, 30191, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 492, 36563, 198, 220, 220, 220, 220, 220, 220, 220, 16326, 13, 2213, 19524, 378, 7, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 628, 220, 220, 220, 220, 220, 220, 220, 3373, 16926, 46, 25, 356, 815, 1949, 7069, 284, 1064, 3224, 8245, 16326, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 770, 561, 4646, 262, 2033, 286, 1005, 4233, 1710, 618, 13213, 1224, 45240, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 33973, 284, 262, 9156, 287, 281, 4697, 2196, 286, 428, 2393, 628, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 1224, 45240, 796, 21059, 7248, 3712, 3605, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 611, 21354, 62, 33661, 13, 271, 62, 28920, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1441, 357, 83, 482, 641, 11, 1224, 45240, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 3373, 770, 318, 262, 2562, 1643, 13, 2329, 1064, 477, 2060, 16326, 326, 923, 351, 21354, 62, 33661, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 357, 8201, 16326, 326, 3446, 2872, 21354, 62, 33661, 8, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 8621, 283, 803, 428, 422, 262, 9052, 2174, 5419, 351, 2854, 287, 257, 2219, 1339, 13, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 966, 796, 2116, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 82, 9741, 62, 30001, 62, 33661, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 3911, 653, 62, 4122, 7, 91, 87, 91, 2124, 13, 292, 62, 48369, 3419, 1279, 21354, 62, 33661, 13, 292, 62, 48369, 35430, 198, 220, 220, 220, 220, 220, 220, 220, 981, 966, 1279, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 13, 11925, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 11405, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 301, 5889, 62, 4480, 39434, 403, 31284, 62, 33661, 8, 198, 220, 220, 220, 220, 220, 220, 220, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1224, 45240, 13, 28463, 7, 35138, 0, 58, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2116, 13, 12685, 12342, 58, 944, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 292, 62, 48369, 3419, 4357, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2361, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 966, 15853, 352, 26, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 3373, 2735, 4174, 772, 517, 33908, 2700, 13, 1629, 790, 357, 847, 8, 1744, 2292, 329, 262, 965, 2860, 1359, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 11241, 11, 1673, 36686, 378, 3224, 9881, 422, 326, 11241, 357, 361, 597, 8, 284, 21354, 62, 33661, 11, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 290, 1005, 4233, 786, 262, 2187, 1517, 290, 766, 644, 356, 651, 13, 198, 220, 220, 220, 220, 220, 220, 220, 329, 1312, 287, 352, 492, 403, 31284, 62, 33661, 13, 11925, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 21231, 796, 1222, 403, 31284, 62, 33661, 58, 492, 72, 11208, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 35488, 796, 1222, 403, 31284, 62, 33661, 58, 72, 492, 11208, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 966, 796, 2116, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 82, 9741, 62, 30001, 62, 33661, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 3911, 653, 62, 4122, 7, 91, 87, 91, 2124, 13, 292, 62, 48369, 3419, 1279, 35488, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 16926, 46, 25, 2448, 69, 6436, 5612, 611, 35488, 4940, 351, 366, 366, 30, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 981, 966, 1279, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 13, 11925, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 11405, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 301, 5889, 62, 4480, 7, 37333, 844, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 5885, 796, 685, 40290, 11, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 58, 4122, 4083, 292, 62, 48369, 3419, 4083, 1102, 9246, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 30240, 796, 2872, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 39434, 79, 43691, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 3461, 453, 11, 428, 318, 18022, 62, 24874, 62, 268, 8189, 39434, 79, 43691, 11, 1222, 944, 13, 12685, 12342, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 887, 356, 1244, 423, 5495, 257, 40364, 6626, 543, 561, 2948, 4017, 3212, 13, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 357, 31722, 1744, 287, 262, 4931, 286, 21354, 40364, 30778, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 1406, 10385, 284, 41002, 12, 23, 290, 466, 40364, 26021, 13, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 412, 13, 70, 13, 351, 537, 3064, 74, 62, 8692, 366, 220, 220, 2474, 3011, 6626, 284, 366, 366, 1343, 366, 5145, 1600, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 475, 18022, 62, 24874, 62, 268, 8189, 7203, 220, 220, 2474, 8, 14512, 18022, 62, 24874, 62, 268, 8189, 7203, 366, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 6762, 7, 82, 8, 5218, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 82, 828, 628, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 5429, 1146, 11, 1771, 393, 407, 428, 3211, 318, 3376, 8338, 319, 1771, 612, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 561, 307, 257, 40364, 6626, 878, 262, 41002, 12, 23, 40122, 341, 966, 13, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 18578, 21404, 1576, 326, 645, 530, 481, 1683, 4003, 357, 8499, 477, 11, 661, 1422, 470, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 4003, 477, 262, 1263, 10421, 287, 262, 2180, 21354, 11241, 7822, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 41512, 28264, 8, 5218, 18022, 62, 24874, 62, 268, 8189, 39434, 79, 43691, 11, 1222, 944, 13, 12685, 12342, 828, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 13742, 588, 262, 1708, 318, 19827, 475, 11491, 25, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 41512, 7, 68, 8, 5218, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 13271, 8635, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 220, 220, 220, 220, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 62, 403, 26752, 39434, 79, 43691, 58, 492, 68, 13, 12102, 62, 929, 62, 1462, 3419, 12962, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 1782, 828, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 18083, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 33756, 796, 38692, 3712, 3605, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 33756, 62, 11925, 796, 657, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 329, 11241, 287, 30240, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 33756, 13, 14689, 7, 30001, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 33756, 62, 11925, 15853, 2116, 13, 12501, 12342, 58, 5, 30001, 4083, 11925, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 33756, 62, 11925, 18189, 21354, 62, 33661, 13, 11925, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2270, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1224, 45240, 13, 28463, 7, 41068, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 966, 15853, 352, 26, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 3373, 770, 318, 635, 407, 15836, 13, 2893, 356, 4143, 7048, 326, 40364, 30778, 389, 8245, 11, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 12716, 11, 484, 389, 407, 13, 1320, 318, 11, 611, 4375, 9881, 547, 284, 787, 257, 6626, 1656, 287, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 21354, 62, 33661, 11, 428, 714, 787, 16326, 1744, 543, 674, 9156, 561, 4306, 892, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 561, 307, 23791, 13, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 1114, 1672, 11, 351, 308, 457, 17, 11, 262, 779, 286, 3467, 82, 33747, 12248, 59, 50, 8, 1724, 326, 37082, 77, 59, 77, 1, 714, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 1205, 257, 6626, 11, 304, 13, 70, 13, 37082, 77, 59, 77, 15, 1, 30778, 656, 37082, 77, 1, 10, 1, 59, 77, 1, 10, 1, 15, 1600, 1642, 37082, 77, 1, 257, 1744, 11241, 13, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 3423, 318, 257, 2068, 290, 11841, 4259, 25, 198, 220, 220, 220, 220, 220, 220, 220, 3373, 770, 2125, 470, 826, 611, 356, 1683, 4781, 3467, 82, 33747, 12248, 59, 50, 8, 198, 220, 220, 220, 220, 220, 220, 220, 611, 21354, 62, 33661, 13, 11925, 3419, 1875, 352, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 938, 62, 12501, 9043, 796, 275, 2536, 3712, 12501, 1098, 62, 12957, 62, 40477, 23, 7, 403, 31284, 62, 33661, 13, 292, 62, 48369, 35430, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 21354, 62, 33661, 13, 11925, 3419, 532, 938, 62, 12501, 9043, 13, 16, 1875, 657, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 11405, 938, 62, 12501, 9043, 13, 15, 13, 8899, 62, 273, 7, 9562, 11, 930, 66, 91, 269, 13, 271, 62, 1929, 2737, 10223, 28955, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 302, 12685, 9043, 796, 18022, 62, 24874, 62, 268, 8189, 7, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1222, 403, 31284, 62, 33661, 58, 492, 403, 31284, 62, 33661, 13, 11925, 3419, 532, 938, 62, 12501, 9043, 13, 16, 4357, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1222, 944, 13, 12685, 12342, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 5619, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 302, 12685, 9043, 13, 2302, 437, 7, 26327, 62, 24874, 62, 268, 8189, 7, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1222, 403, 31284, 62, 33661, 58, 403, 31284, 62, 33661, 13, 11925, 3419, 532, 938, 62, 12501, 9043, 13, 16, 492, 4357, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1222, 944, 13, 12685, 12342, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 29226, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1224, 45240, 13, 28463, 7, 1361, 40976, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 628, 220, 220, 220, 220, 220, 220, 220, 357, 83, 482, 641, 11, 1224, 45240, 8, 198, 220, 220, 220, 1782, 198, 92, 198, 198, 2, 58, 9078, 24396, 82, 60, 198, 23928, 7231, 33, 11401, 1391, 198, 220, 220, 220, 1303, 58, 3605, 60, 198, 220, 220, 220, 24714, 649, 7, 198, 220, 220, 220, 220, 220, 220, 220, 2207, 12342, 25, 21059, 13912, 27, 53, 721, 27, 84, 23, 22330, 514, 1096, 22330, 198, 220, 220, 220, 220, 220, 220, 220, 2041, 62, 83, 482, 641, 62, 12685, 12342, 25, 21059, 13912, 27, 10100, 11, 514, 1096, 22330, 198, 220, 220, 220, 220, 220, 220, 220, 3912, 25, 1222, 2536, 11, 198, 220, 220, 220, 1267, 4613, 9485, 23004, 27, 24704, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 40364, 796, 797, 25636, 3712, 3605, 7, 33279, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 8899, 62, 8056, 7, 91, 68, 91, 9485, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 11395, 12331, 11, 4808, 33994, 68, 13, 1462, 62, 8841, 3419, 4008, 30, 26, 628, 220, 220, 220, 220, 220, 220, 220, 1309, 2041, 62, 260, 25636, 796, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4808, 42632, 796, 2041, 62, 83, 482, 641, 62, 12685, 12342, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 13083, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 8899, 7, 91, 82, 91, 14996, 62, 260, 25636, 3712, 41915, 7, 82, 4008, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 33327, 3712, 27, 53, 721, 27, 62, 4211, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 797, 25636, 3712, 3605, 39434, 62, 42632, 13, 22179, 7203, 91, 48774, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 8899, 62, 8056, 7, 91, 68, 91, 9485, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 11395, 12331, 11, 4808, 33994, 68, 13, 1462, 62, 8841, 3419, 4008, 30, 198, 220, 220, 220, 220, 220, 220, 220, 18083, 628, 220, 220, 220, 220, 220, 220, 220, 1309, 875, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 796, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2207, 12342, 13, 2676, 22446, 8899, 7, 91, 7, 74, 11, 410, 14726, 20789, 85, 11, 479, 13, 21018, 28955, 737, 33327, 9783, 628, 220, 220, 220, 220, 220, 220, 220, 6818, 0, 7, 12685, 12342, 13, 11925, 3419, 6624, 875, 12342, 13, 11925, 35430, 628, 220, 220, 220, 220, 220, 220, 220, 1309, 2041, 62, 83, 482, 641, 62, 12501, 12342, 25, 21059, 13912, 27, 385, 1096, 11, 38692, 27, 84, 23, 4211, 796, 2041, 62, 83, 482, 641, 62, 12685, 12342, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 2676, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 8899, 7, 91, 7, 74, 11, 410, 14726, 20789, 85, 11, 479, 13, 292, 62, 33661, 22446, 1462, 62, 35138, 3419, 4008, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 33327, 9783, 628, 220, 220, 220, 220, 220, 220, 220, 3373, 30698, 780, 314, 836, 470, 760, 703, 284, 1560, 17103, 314, 1101, 407, 1016, 284, 1487, 262, 3975, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 23243, 62, 30001, 62, 33661, 25, 38692, 27, 53, 721, 27, 84, 23, 4211, 796, 2207, 12342, 13, 13083, 22446, 565, 12004, 22446, 33327, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 23243, 62, 30001, 62, 33661, 13, 30619, 9783, 628, 220, 220, 220, 220, 220, 220, 220, 6762, 7, 14055, 33, 11401, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2207, 12342, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2041, 62, 83, 482, 641, 62, 12685, 12342, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 875, 12342, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2041, 62, 83, 482, 641, 62, 12501, 12342, 11, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 40364, 62, 83, 7278, 25, 357, 15, 492, 22921, 62, 41359, 62, 4221, 15675, 50, 737, 8899, 7, 91, 62, 91, 40364, 13, 21018, 3419, 737, 33327, 22784, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2041, 62, 260, 25636, 62, 83, 7278, 25, 357, 15, 492, 22921, 62, 41359, 62, 4221, 15675, 50, 8, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 8899, 7, 91, 62, 91, 2041, 62, 260, 25636, 13, 21018, 28955, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 33327, 22784, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 23243, 62, 30001, 62, 33661, 11, 198, 220, 220, 220, 220, 220, 220, 220, 32092, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 3373, 36658, 18604, 198, 220, 220, 220, 3373, 14711, 7656, 198, 220, 220, 220, 3373, 36658, 18604, 628, 220, 220, 220, 24714, 37773, 62, 35947, 39434, 944, 11, 12972, 25, 11361, 11, 2420, 25, 1222, 2536, 8, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 5239, 4008, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 37773, 39434, 944, 11, 12972, 25, 11361, 11, 2420, 25, 1222, 2536, 11, 3142, 62, 20887, 25, 21059, 7248, 27, 5, 2536, 43734, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 268, 8189, 62, 30191, 7, 5239, 11, 1222, 40845, 62, 20887, 737, 15, 8, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 4808, 268, 8189, 62, 33661, 39434, 944, 11, 12972, 25, 11361, 11, 9881, 25, 1222, 58, 84, 23, 12962, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2872, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 7, 33661, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 6762, 7, 5239, 8, 5218, 2116, 13557, 268, 8189, 62, 35947, 62, 30191, 7, 5239, 828, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 41512, 7, 68, 8, 5218, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 2420, 796, 21596, 1391, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 62, 403, 26752, 39434, 33661, 58, 492, 68, 13, 12102, 62, 929, 62, 1462, 3419, 12962, 18083, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 357, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 2116, 13557, 268, 8189, 62, 30191, 7, 5239, 11, 1222, 26257, 7248, 3712, 3605, 35430, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 357, 21973, 16326, 11, 938, 62, 12239, 62, 30001, 62, 11925, 8, 796, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2116, 13557, 24988, 589, 62, 12957, 62, 12239, 62, 30001, 62, 11925, 7, 83, 482, 641, 11, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 5145, 83, 482, 641, 13, 271, 62, 28920, 3419, 11405, 938, 62, 12239, 62, 30001, 62, 11925, 1875, 657, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 406, 404, 572, 262, 16326, 422, 262, 938, 3704, 290, 1057, 347, 11401, 319, 262, 5637, 9881, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 32499, 5183, 21404, 11, 475, 428, 743, 407, 307, 3376, 611, 356, 1549, 423, 550, 257, 40364, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 6626, 1022, 262, 4938, 41002, 12, 23, 290, 262, 12515, 9881, 11, 543, 318, 1521, 428, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 3373, 2446, 318, 2839, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 21354, 62, 33661, 796, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 2116, 13557, 12501, 1098, 62, 30191, 39434, 83, 482, 641, 58, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 492, 36563, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 21354, 62, 33661, 13, 2302, 437, 62, 6738, 62, 48369, 39434, 33661, 58, 68, 13, 12102, 62, 929, 62, 1462, 3419, 492, 36563, 628, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 16326, 13, 2213, 19524, 378, 7, 83, 482, 641, 13, 11925, 3419, 532, 938, 62, 12239, 62, 30001, 62, 11925, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 16326, 13, 2302, 437, 7, 26327, 62, 24874, 62, 268, 8189, 39434, 403, 31284, 62, 33661, 11, 1222, 944, 13, 12685, 12342, 18125, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 16326, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 32092, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 37773, 62, 4480, 62, 403, 31284, 7, 198, 220, 220, 220, 220, 220, 220, 220, 1222, 944, 11, 198, 220, 220, 220, 220, 220, 220, 220, 12972, 25, 11361, 11, 198, 220, 220, 220, 220, 220, 220, 220, 2420, 25, 1222, 2536, 11, 198, 220, 220, 220, 220, 220, 220, 220, 3142, 62, 20887, 25, 21059, 7248, 27, 5, 2536, 22330, 198, 220, 220, 220, 1267, 4613, 9485, 27, 20519, 51, 29291, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 357, 83, 482, 641, 11, 1224, 45240, 8, 796, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 268, 8189, 62, 403, 31284, 62, 30191, 7, 5239, 11, 1222, 40845, 62, 20887, 18125, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 12972, 62, 785, 37069, 507, 796, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 9485, 8053, 3712, 3605, 7, 9078, 11, 1224, 45240, 13, 2676, 22446, 8899, 7, 91, 41068, 91, 9485, 8053, 3712, 3605, 7, 9078, 11, 1222, 41068, 58, 492, 60, 4008, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 357, 83, 482, 641, 11, 12972, 62, 785, 37069, 507, 737, 20424, 62, 9078, 7, 9078, 8, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 37773, 62, 29762, 62, 30001, 39434, 944, 11, 3704, 25, 1222, 58, 84, 23, 12962, 4613, 9485, 23004, 27, 385, 1096, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 737, 22163, 798, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1441, 6762, 7, 30001, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 6762, 7, 12239, 62, 2536, 8, 796, 14367, 3712, 2536, 3712, 6738, 62, 40477, 23, 7, 12239, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 20887, 62, 83, 482, 641, 62, 12685, 12342, 13, 1136, 7, 12239, 62, 2536, 737, 22163, 798, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1441, 6762, 7, 30001, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 41512, 7, 20519, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 9218, 12331, 11, 4808, 33994, 12239, 13, 1462, 62, 11990, 3419, 4008, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 37773, 62, 29762, 62, 12239, 39434, 944, 11, 3704, 25, 1222, 58, 84, 23, 12962, 4613, 38692, 27, 385, 1096, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 2773, 7, 30001, 8, 796, 2116, 13, 12685, 12342, 13, 1136, 7, 12239, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1441, 43030, 0, 58, 9, 30001, 11208, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 18022, 62, 24874, 62, 268, 8189, 7, 12239, 11, 1222, 944, 13, 12685, 12342, 8, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 3373, 36658, 18604, 198, 220, 220, 220, 3373, 4280, 7656, 198, 220, 220, 220, 3373, 36658, 18604, 628, 220, 220, 220, 24714, 36899, 62, 33661, 39434, 944, 11, 12972, 25, 11361, 11, 16326, 25, 38692, 27, 385, 1096, 43734, 4613, 9485, 27, 20519, 45992, 29, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 9881, 796, 12972, 13, 12154, 62, 16663, 82, 7, 15886, 2116, 13557, 12501, 1098, 62, 30191, 39434, 83, 482, 641, 18125, 198, 220, 220, 220, 220, 220, 220, 220, 9485, 45992, 3712, 3605, 7, 9078, 11, 1222, 33661, 737, 20424, 3419, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 24714, 36899, 62, 29762, 62, 30001, 62, 33661, 39434, 944, 11, 12972, 25, 11361, 11, 11241, 25, 514, 1096, 8, 4613, 9485, 23004, 27, 20519, 27, 20519, 45992, 4211, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 2773, 7, 33661, 8, 796, 2116, 13, 12501, 12342, 13, 1136, 39434, 30001, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1441, 6762, 7, 20519, 45992, 3712, 3605, 7, 9078, 11, 9881, 737, 20424, 35430, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 611, 1309, 2773, 7, 33661, 8, 796, 2116, 13, 20887, 62, 83, 482, 641, 62, 12501, 12342, 13, 1136, 39434, 30001, 8, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 1441, 6762, 7, 20519, 45992, 3712, 3605, 7, 9078, 11, 9881, 737, 20424, 35430, 198, 220, 220, 220, 220, 220, 220, 220, 1782, 198, 220, 220, 220, 220, 220, 220, 220, 41512, 7, 20519, 9139, 81, 3712, 3605, 3712, 27, 1069, 11755, 3712, 20519, 9218, 12331, 11, 4808, 33994, 30001, 13, 1462, 62, 8841, 3419, 4008, 198, 220, 220, 220, 1782, 628, 220, 220, 220, 3373, 36658, 18604, 198, 220, 220, 220, 3373, 46253, 198, 220, 220, 220, 3373, 36658, 18604, 628, 220, 220, 220, 24714, 11241, 62, 26327, 62, 27160, 39434, 944, 11, 12972, 25, 11361, 8, 4613, 38692, 27, 20519, 27, 20519, 45992, 4211, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 2116, 13, 82, 9741, 62, 30001, 62, 33661, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 2676, 3419, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 8899, 7, 91, 87, 91, 9485, 45992, 3712, 3605, 7, 9078, 11, 2124, 737, 20424, 28955, 198, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 220, 764, 33327, 3419, 198, 220, 220, 220, 1782, 198, 92, 198, 198, 2, 58, 9078, 21412, 60, 198, 22184, 4808, 83, 1134, 30001, 28264, 9078, 25, 11361, 11, 285, 25, 1222, 20519, 26796, 8, 4613, 9485, 23004, 27, 3419, 29, 1391, 198, 220, 220, 220, 285, 13, 2860, 62, 4871, 3712, 27, 14055, 33, 11401, 29, 3419, 30, 26, 198, 220, 220, 220, 6762, 7, 28955, 198, 92, 198, 198, 2, 58, 37581, 7, 9288, 15437, 198, 4666, 5254, 1391, 198, 220, 220, 220, 779, 17000, 66, 62, 17831, 3712, 37, 87, 26257, 13912, 355, 21059, 13912, 26, 628, 220, 220, 220, 779, 27021, 3712, 26327, 62, 24874, 62, 35312, 26, 628, 220, 220, 220, 1303, 58, 9288, 60, 198, 220, 220, 220, 24714, 845, 62, 36439, 62, 9288, 3419, 1391, 198, 220, 220, 220, 220, 220, 220, 220, 1309, 4517, 9803, 796, 21059, 13912, 3712, 12286, 9783, 198, 220, 220, 220, 220, 220, 220, 220, 9803, 13, 28463, 7, 65, 1, 397, 1911, 1462, 62, 35138, 22784, 352, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 9803, 13, 28463, 7, 65, 1, 10210, 1911, 1462, 62, 35138, 22784, 362, 1776, 628, 220, 220, 220, 220, 220, 220, 220, 1309, 581, 796, 18022, 62, 24874, 62, 35312, 7, 65, 1, 397, 10210, 1600, 1222, 81, 2283, 1776, 198, 220, 220, 220, 220, 220, 220, 220, 6818, 62, 27363, 0, 7, 411, 11, 43030, 0, 58, 65, 1, 397, 1600, 275, 1, 10210, 8973, 1776, 198, 220, 220, 220, 1782, 198, 92, 198] \ No newline at end of file diff --git a/test/Microsoft.ML.Tokenizers.Tests/Microsoft.ML.Tokenizers.Tests.csproj b/test/Microsoft.ML.Tokenizers.Tests/Microsoft.ML.Tokenizers.Tests.csproj index 65fedd6de1..34497f8e20 100644 --- a/test/Microsoft.ML.Tokenizers.Tests/Microsoft.ML.Tokenizers.Tests.csproj +++ b/test/Microsoft.ML.Tokenizers.Tests/Microsoft.ML.Tokenizers.Tests.csproj @@ -23,7 +23,28 @@ - + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + + + \ No newline at end of file diff --git a/test/Microsoft.ML.Tokenizers.Tests/PreTokenizerTests.cs b/test/Microsoft.ML.Tokenizers.Tests/PreTokenizerTests.cs index d14ef6f3c3..b35b2ea83b 100644 --- a/test/Microsoft.ML.Tokenizers.Tests/PreTokenizerTests.cs +++ b/test/Microsoft.ML.Tokenizers.Tests/PreTokenizerTests.cs @@ -65,12 +65,12 @@ public void TestPreTokenizer(PreTokenizer preTokenizer, string sentence, Split[] [Fact] public void TestWhiteSpacePreTokenizer() { - WhiteSpace.Instance.PreTokenize(null); + Assert.Empty(WhiteSpace.Instance.PreTokenize(null!)); } public class SpacePreTokenizer : PreTokenizer { - public override IReadOnlyList PreTokenize(string sentence) + public override IEnumerable PreTokenize(string sentence, bool skipSpecialTokens = false) { List splits = new(); if (string.IsNullOrEmpty(sentence)) diff --git a/test/Microsoft.ML.Tokenizers.Tests/TitokenTests.cs b/test/Microsoft.ML.Tokenizers.Tests/TitokenTests.cs new file mode 100644 index 0000000000..3e2b2913f6 --- /dev/null +++ b/test/Microsoft.ML.Tokenizers.Tests/TitokenTests.cs @@ -0,0 +1,208 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.ML.Tokenizers; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using Xunit; + +namespace Microsoft.ML.Tokenizers.Tests +{ + public class TiktokenTests + { + const string IMStart = "<|im_start|>"; + const string IMEnd = "<|im_end|>"; + + private static readonly Dictionary _specialTokens = new Dictionary + { + { IMStart, 100264}, + { IMEnd, 100265}, + }; + + public static Tokenizer GPT4 { get; } = Tokenizer.CreateByModelNameAsync("gpt-4", _specialTokens).GetAwaiter().GetResult(); + public static Tokenizer GPT2 { get; } = Tokenizer.CreateByModelNameAsync("gpt2").GetAwaiter().GetResult(); + public static Tokenizer P50kBase { get; } = Tokenizer.CreateByModelNameAsync("text-davinci-003").GetAwaiter().GetResult(); + public static Tokenizer R50kBase { get; } = Tokenizer.CreateByModelNameAsync("ada").GetAwaiter().GetResult(); + public static Tokenizer P50kEdit { get; } = Tokenizer.CreateByModelNameAsync("text-davinci-edit-001").GetAwaiter().GetResult(); + + [Fact] + public void TestGPT4TokenizationEncoding() + { + string text = "Hello World"; + IReadOnlyList encoded = GPT4.EncodeToIds(text); + Assert.Equal(new List() { 9906, 4435 }, encoded); + Assert.Equal(text, GPT4.Decode(encoded.ToArray())!); + + TokenizerResult result = GPT4.Encode(text); + Assert.Equal(new List() { 9906, 4435 }, result.Ids); + Assert.Equal(new string[] { "Hello", " World" }, result.Tokens); + Assert.Equal(new List<(int, int)> { (0, 5), (5, 11) }, result.Offsets); + } + + [Fact] + public void TestEncode1() + { + var text = "<|im_start|>Hello World<|im_end|>"; + IReadOnlyList encoded = GPT4.EncodeToIds(text); + Assert.Equal(new List() { 100264, 9906, 4435, 100265 }, encoded); + Assert.Equal(text, GPT4.Decode(encoded.ToArray())); + + TokenizerResult result = GPT4.Encode(text); + Assert.Equal(new List() { 100264, 9906, 4435, 100265 }, result.Ids); + Assert.Equal(new string[] { "<|im_start|>", "Hello", " World", "<|im_end|>" }, result.Tokens); + Assert.Equal(new List<(int, int)> { (0, 12), (12, 17), (17, 23), (23, 33) }, result.Offsets); + } + + [Fact] + public void TestEncode2() + { + string text = ReadAndSanitizeFile("./Data/lib.rs.txt"); + IReadOnlyList encoded = GPT4.EncodeToIds(text, skipSpecialTokens: true); + Assert.Equal(5584, encoded.Count); + + using (Stream stream = File.OpenRead("./Data/tokens.json")) + { + int[]? expected = JsonSerializer.Deserialize(stream) as int[]; + Assert.Equal(expected!, encoded.ToArray()); + } + + string? decoded = GPT4.Decode(encoded.ToArray()); + Assert.Equal(text, decoded!); + } + + [Fact] + public void TestEncode3() + { + string text = "<|im_start|>Hello<|im_end|> World"; + IReadOnlyList encoded = GPT4.EncodeToIds(text); + Assert.Equal(new List() { 100264, 9906, 100265, 4435 }, encoded); + string? decoded = GPT4.Decode(encoded.ToArray()); + Assert.Equal(text, decoded); + + TokenizerResult result = GPT4.Encode(text); + Assert.Equal(new List() { 100264, 9906, 100265, 4435 }, result.Ids); + Assert.Equal(new string[] { "<|im_start|>", "Hello", "<|im_end|>", " World" }, result.Tokens); + Assert.Equal(new List<(int, int)> { (0, 12), (12, 17), (17, 27), (27, 33) }, result.Offsets); + } + + [Fact] + public void TestEncode4() + { + string text = ""; + IReadOnlyList encoded = GPT4.EncodeToIds(text); + Assert.Empty(encoded); + + TokenizerResult result = GPT4.Encode(text); + Assert.Empty(result.Ids); + Assert.Empty(result.Tokens); + Assert.Empty(result.Offsets); + } + + [Fact] + public void TestEncode5() + { + string text = "<|im_start|>Hello ⭐ World<|im_end|>"; + IReadOnlyList encoded = GPT4.EncodeToIds(text); + Assert.Equal(new List() { 100264, 9906, 2928, 99834, 4435, 100265 }, encoded); + Assert.Equal(text, GPT4.Decode(encoded.ToArray())); + + TokenizerResult result = GPT4.Encode(text); + Assert.Equal(new List() { 100264, 9906, 2928, 99834, 4435, 100265 }, result.Ids); + Assert.Equal(new string[] { "<|im_start|>", "Hello", " ⭐", "", " World", "<|im_end|>" }, result.Tokens); + Assert.Equal(new List<(int, int)> { (0, 12), (12, 17), (17, 19), (19, 19), (19, 25), (25, 35) }, result.Offsets); + } + + [Fact] + public void TestEncodeGpt2() + { + string text = ReadAndSanitizeFile("./Data/lib.rs.txt"); + IReadOnlyList encoded = GPT2.EncodeToIds(text); + Assert.Equal(11378, encoded.Count); + + using (Stream stream = File.OpenRead("./Data/tokens_gpt2.json")) + { + int[]? expected = JsonSerializer.Deserialize(stream) as int[]; + Assert.Equal(expected!, encoded.ToArray()); + } + + string? decoded = GPT2.Decode(encoded.ToArray()); + Assert.Equal(text, decoded); + } + + [Fact] + public void TestEncodeP50kBase() + { + string text = ReadAndSanitizeFile("./Data/lib.rs.txt"); + IReadOnlyList encoded = P50kBase.EncodeToIds(text); + Assert.Equal(7230, encoded.Count); + + using (Stream stream = File.OpenRead("./Data/tokens_p50k_base.json")) + { + int[]? expected = JsonSerializer.Deserialize(stream) as int[]; + Assert.Equal(expected!, encoded.ToArray()); + } + + string? decoded = P50kBase.Decode(encoded.ToArray()); + Assert.Equal(text, decoded); + } + + [Fact] + public void TestEncodeP50kEdit() + { + string text = ReadAndSanitizeFile("./Data/lib.rs.txt"); + IReadOnlyList encoded = P50kEdit.EncodeToIds(text); + Assert.Equal(7230, encoded.Count); + + using (Stream stream = File.OpenRead("./Data/tokens_p50k_edit.json")) + { + int[]? expected = JsonSerializer.Deserialize(stream) as int[]; + Assert.Equal(expected!, encoded.ToArray()); + } + + string? decoded = P50kEdit.Decode(encoded.ToArray()); + Assert.Equal(text, decoded); + } + + [Fact] + public void TestEncodeR50kBase() + { + string text = ReadAndSanitizeFile("./Data/lib.rs.txt"); + IReadOnlyList encoded = R50kBase.EncodeToIds(text); + Assert.Equal(11378, encoded.Count); + + using (Stream stream = File.OpenRead("./Data/tokens_r50k_base.json")) + { + int[]? expected = JsonSerializer.Deserialize(stream) as int[]; + Assert.Equal(expected!, encoded.ToArray()); + } + + string? decoded = R50kBase.Decode(encoded.ToArray()); + Assert.Equal(text, decoded); + } + + // Test running copy the test data files to the output folder but sometimes the file content is mutated replacing '\n' with '\r\n'. + // This method reads the file and removes the extra inserted '\r' characters. Having '\r' in the file content will cause the tests to fail. + private string ReadAndSanitizeFile(string path) + { + // Didn't use String.Replace because the version accept stringComparison parameter is not supported on NETFX. + string text = File.ReadAllText(path); + StringBuilder sb = new StringBuilder(); + + foreach (char c in text) + { + if (c != '\r') + { + sb.Append(c); + } + } + return sb.ToString(); + } + } +} +