diff --git a/predictMaskedToken.m b/predictMaskedToken.m index 45a8452..e9b4a79 100644 --- a/predictMaskedToken.m +++ b/predictMaskedToken.m @@ -42,7 +42,7 @@ for i = 1:numel(pieces) tokens = fulltok.tokenize(pieces(i)); - if ~isempty(tokens) + if ~isempty(tokens{1}) % "" tokenizes to empty - awkward x = cat(2,x,fulltok.encode(tokens{1})); end diff --git a/test/tpredictMaskedToken.m b/test/tpredictMaskedToken.m index b21ceac..5a577ff 100644 --- a/test/tpredictMaskedToken.m +++ b/test/tpredictMaskedToken.m @@ -9,7 +9,7 @@ ValidText = iGetValidText; end - methods(Test) + methods(Test) function verifyOutputDimSizes(test, Models, ValidText) inSize = size(ValidText); mdl = bert("Model", Models); @@ -43,5 +43,8 @@ function inputWithoutMASKRemainsTheSame(test, Models) validText = struct('StringsAsColumns',manyStrs,... 'StringsAsRows',manyStrs',... 'ManyStrings',repmat(singleStr,3),... - 'SingleString',singleStr); + 'SingleString',singleStr,... + 'FirstWordMasked', "[MASK] the pace.",... + 'LastWordMasked', "Accelerating the [MASK].",... + 'SingleMaskToken', "[MASK]"); end \ No newline at end of file