Add support for writing Windows-1252

Author: nexos-dev

.editorconfig

@@ -27,7 +27,7 @@ charset = utf-8
 insert_final_newline = true
 [*.testxt]
 charset = unset
-end_of_line = lf
+end_of_line = unset
 insert_final_newline = true
 indent_style = space
 indent_size = 4

include/libnex/base.h

@@ -0,0 +1,30 @@
+/*
+    base.h - contains macros that handle base stuff
+    Copyright 2022 The NexNix Project
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    There should be a copy of the License distributed in a file named
+    LICENSE, if not, you may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+/// @file base.h
+
+#ifndef _BASE_H
+#define _BASE_H
+
+// Used to specify that a parameter is unused
+#define UNUSED(param) (void) (param);
+
+// To find the size of an array
+#define ARRAY_SIZE(a) (sizeof (a) / sizeof ((a)[0]))
+
+#endif

src/libnex_baremetal.h

@@ -26,6 +26,4 @@
 #include <libnex/endian.h>
 #include <libnex/safestring.h>
 
-#define UNUSED(param) (void) (param);
-
 #endif

src/libnex_hosted.h

@@ -31,6 +31,4 @@
 #include <libnex/safestring.h>
 #include <libnex/textstream.h>
 
-#define UNUSED(param) (void) (param);
-
 #endif

src/textstream.c

@@ -20,6 +20,7 @@
 
 #include "codepages/win1252toUtf32.h"
 #include <errno.h>
+#include <libnex/base.h>
 #include <libnex/bits.h>
 #include <libnex/safemalloc.h>
 #include <libnex/textstream.h>
@@ -175,7 +176,7 @@ PUBLIC void TextSetBufSz (TextStream_t* stream, size_t sz)
 }
 
 // Checks if we reached a newline
-char _textCheckNewLine (wchar_t* buf, int i, size_t bytesLeft)
+char _textCheckNewLine (uint8_t* buf, int i, size_t bytesLeft)
 {
     if (buf[i] == '\n' || buf[i] == '\r')
     {
@@ -203,7 +204,7 @@ ssize_t _textDecode (TextStream_t* stream, wchar_t* buf, size_t count, int termi
             // Check if a terminator was reached
             if (terminator == 1)
             {
-                char doWhat = _textCheckNewLine (buf, i, count);
+                char doWhat = _textCheckNewLine (stream->buf, i, count);
                 if (doWhat == 0)
                     continue;
                 else if (doWhat == 1)
@@ -234,7 +235,7 @@ ssize_t _textDecode (TextStream_t* stream, wchar_t* buf, size_t count, int termi
             // Check if a terminator was reached
             if (terminator == 1)
             {
-                char doWhat = _textCheckNewLine (buf, i, count);
+                char doWhat = _textCheckNewLine (stream->buf, i, count);
                 if (doWhat == 0)
                     continue;
                 else if (doWhat == 1)
@@ -264,8 +265,43 @@ ssize_t _textEncode (TextStream_t* stream, wchar_t* buf, size_t count)
                 return -1;
             }
             stream->buf[i] = (uint8_t) buf[i];
-            charEncoded = (ssize_t) count;
         }
+        charEncoded = (ssize_t) count;
+    }
+    else if (stream->encoding == TEXT_ENC_WIN1252)
+    {
+        // Loop and encode
+        for (int i = 0; i < count; ++i)
+        {
+            // This is where the algorithm starts. Check if this character's Unicode code
+            // is the same as its Windows-1252 one. If it is, directly copy to destination
+            // buffer
+            if (buf[i] <= 0x7F || (buf[i] >= 0xA0 && buf[i] <= 0xFF))
+            {
+                // Copy out
+                stream->buf[i] = (uint8_t) buf[i];
+            }
+            // It's a Windows-1252 character
+            else
+            {
+                // This is kind of slow, but the best way overall.
+                // We loop through the translation table until we find character that matches
+                // buf[i]. We set bit 7 on the index, and that's the character
+                int tableSize = ARRAY_SIZE (win1252toUtf32);
+                int tableIndex = 0;
+                while (tableIndex < tableSize)
+                {
+                    // Check for a match
+                    if (win1252toUtf32[tableIndex] == buf[i])
+                    {
+                        // Set bit 7 on tableIndex, and that is the character
+                        stream->buf[i] = BitSetNew (tableIndex, 7);
+                    }
+                    tableIndex++;
+                }
+            }
+        }
+        charEncoded = (ssize_t) count;
     }
     return charEncoded;
 }
@@ -303,7 +339,7 @@ PUBLIC ssize_t TextRead (TextStream_t* stream, wchar_t* buf, size_t count)
     // Read the data into the staging buffer
     ssize_t charRead = (ssize_t) fread (stream->buf, 1, count * sizeof (wchar_t), stream->file);
     // Decode the string
-    ssize_t charParsed = _textDecode (stream, buf, charRead, 0);
+    ssize_t charParsed = _textDecode (stream, buf, charRead, TEXT_DECODE_ALL);
     if (charParsed == -1)
         return -1;
     // That's it
@@ -345,7 +381,7 @@ PUBLIC ssize_t TextReadLine (TextStream_t* stream, wchar_t* buf, size_t count)
     // Read the data into the staging buffer
     ssize_t charRead = (ssize_t) fread (stream->buf, 1, count * sizeof (wchar_t), stream->file);
     // Decode the string
-    ssize_t charParsed = _textDecode (stream, buf, charRead, 1);
+    ssize_t charParsed = _textDecode (stream, buf, charRead, TEXT_DECODE_TERMINATE_ON_NEWLINE);
     // That's it
     TextUnlock (stream);
     return charParsed;

tests/testAscii1.testxt

@@ -1 +1 @@
-Test string. This is an ASCII document.
+Test string. This is an ASCII document.

tests/textstream.c

@@ -37,12 +37,21 @@ int main()
             return 1;
         // FIXME: If C compiler doesn't use unicode by default, these wcscmp's may not work
         // Luckily, CL uses UTF-16 and Clang and GCC use UTF-32, making this work most of the time.
-        wchar_t buf2[] = L"Test string. This is an ASCII document.\n";
+        wchar_t buf2[] = L"Test string. This is an ASCII document.\r\n";
         TEST_BOOL (!wcscmp (buf, buf2), "reading ASCII");
         // Test TextSize
         TEST (TextSize (stream), wcslen (buf2), "TextSize");
         free (buf);
         TextClose (stream);
+        // Test TextReadLine
+        stream = TextOpen ("testAscii1.testxt", TEXT_MODE_READ, TEXT_ENC_ASCII, 0);
+        buf = (wchar_t*) malloc_s (500 * sizeof (wchar_t));
+        if (TextReadLine (stream, buf, 500) == -1)
+            return 1;
+        wchar_t buf5[] = L"Test string. This is an ASCII document.\r\n";
+        TEST_BOOL (!wcscmp (buf, buf5), "reading a line of ASCII");
+        TextClose (stream);
+        free (buf);
         // Write out some text
         wchar_t buf3[] = L"This is a test document.\n";
         // Create a new file
@@ -70,11 +79,28 @@ int main()
             return 1;
         wchar_t* buf = (wchar_t*) malloc_s (500 * sizeof (wchar_t));
         if (TextRead (stream1, buf, 500) == -1)
-            return -1;
+            return 1;
         wchar_t buf2[] = L"Test windows 1252 document. Here is a non-ASCII character: ÿ Ž\n";
         TEST_BOOL (!wcscmp (buf, buf2), "reading Windows 1252");
         free (buf);
         TextClose (stream1);
+        // Test writing it
+        TextStream_t* stream2 = TextOpen ("testWin1252.testout", TEXT_MODE_WRITE, TEXT_ENC_WIN1252, 0);
+        if (!stream2)
+            return 1;
+        if (TextWrite (stream2, buf2, wcslen (buf2)) == -1)
+            return 1;
+        TextClose (stream2);
+        // Read and compare
+        buf = (wchar_t*) malloc_s (500 * sizeof (wchar_t));
+        stream2 = TextOpen ("testWin1252.testout", TEXT_MODE_READ, TEXT_ENC_WIN1252, 0);
+        if (!stream2)
+            return 1;
+        if (TextRead (stream2, buf, wcslen (buf2)) == -1)
+            return 1;
+        TEST_BOOL (!wcscmp (buf, buf2), "writing Windows 1252");
+        TextClose (stream2);
+        free (buf);
     }
     return 0;
 }