Berry added f-strings as an alternative to string formatting

CHANGELOG.md

@@ -13,6 +13,7 @@ All notable changes to this project will be documented in this file.
 - Berry `string.format()` now automatically converts type according to format
 - Matter add friendly-name (NodeLabel) to each endpoint
 - Berry add global function `format` as a simpler syntax to `string.format`
+- Berry added f-strings as an alternative to string formatting
 
 ### Breaking Changed
 

lib/libesp32/berry/src/be_debug.c

@@ -137,7 +137,7 @@ void be_print_inst(binstruction ins, int pc, void* fout)
         logbuf("%s", opc2str(op));
         break;
     }
-    memcpy(__lbuf_tmp, __lbuf, strlen(__lbuf)+1);
+    memcpy(__lbuf_tmp, __lbuf, strlen(__lbuf) + 1);
     logbuf("%s\n", __lbuf_tmp);
     if (fout) {
         be_fwrite(fout, __lbuf, strlen(__lbuf));
@@ -184,7 +184,7 @@ static void sourceinfo(bproto *proto, binstruction *ip)
         blineinfo *end = it + proto->nlineinfo;
         int pc = cast_int(ip - proto->code - 1); /* now vm->ip has been increased */
         for (; it < end && pc > it->endpc; ++it);
-        sprintf(buf, ":%d:", it->linenumber);
+        snprintf(buf, sizeof(buf), ":%d:", it->linenumber);
         be_writestring(str(proto->source));
         be_writestring(buf);
     } else {

lib/libesp32/berry/src/be_exec.c

@@ -183,8 +183,9 @@ int be_protectedparser(bvm *vm,
     return res;
 }
 
-static const char* _sgets(void *data, size_t *size)
+static const char* _sgets(struct blexer* lexer, void *data, size_t *size)
 {
+    (void)lexer;
     struct strbuf *sb = data;
     *size = sb->len;
     if (sb->len) {
@@ -194,8 +195,9 @@ static const char* _sgets(void *data, size_t *size)
     return NULL;
 }
 
-static const char* _fgets(void *data, size_t *size)
+static const char* _fgets(struct blexer* lexer, void *data, size_t *size)
 {
+    (void)lexer;
     struct filebuf *fb = data;
     *size = be_fread(fb->fp, fb->buf, sizeof(fb->buf));
     if (*size) {

lib/libesp32/berry/src/be_lexer.c

@@ -13,14 +13,15 @@
 #include "be_map.h"
 #include "be_vm.h"
 #include "be_strlib.h"
+#include <string.h>
 
 #define SHORT_STR_LEN       32
 #define EOS                 '\0' /* end of source */
 
 #define type_count()        (int)array_count(kwords_tab)
 #define lexbuf(lex)         ((lex)->buf.s)
-#define isvalid(lex)        ((lex)->cursor < (lex)->endbuf)
-#define lgetc(lex)          ((lex)->cursor)
+#define isvalid(lex)        ((lex)->reader.cursor < (lex)->endbuf)
+#define lgetc(lex)          ((lex)->reader.cursor)
 #define setstr(lex, v)      ((lex)->token.u.s = (v))
 #define setint(lex, v)      ((lex)->token.u.i = (v))
 #define setreal(lex, v)     ((lex)->token.u.r = (v))
@@ -39,7 +40,8 @@ static const char* const kwords_tab[] = {
     ":", "?", "->", "if", "elif", "else", "while",
     "for", "def", "end", "class", "break", "continue",
     "return", "true", "false", "nil", "var", "do",
-    "import", "as", "try", "except", "raise", "static"
+    "import", "as", "try", "except", "raise", "static",
+    // ".f"
 };
 
 void be_lexerror(blexer *lexer, const char *msg)
@@ -102,30 +104,34 @@ static int next(blexer *lexer)
     struct blexerreader *lr = &lexer->reader;
     if (!(lr->len--)) {
         static const char eos = EOS;
-        const char *s = lr->readf(lr->data, &lr->len);
+        const char *s = lr->readf(lexer, lr->data, &lr->len);
         lr->s = s ? s : &eos;
         --lr->len;
     }
-    lexer->cursor = *lr->s++;
-    return lexer->cursor;
+    lexer->reader.cursor = *lr->s++;
+    return lexer->reader.cursor;
 }
 
 static void clear_buf(blexer *lexer)
 {
     lexer->buf.len = 0;
 }
 
-/* save and next */
-static int save(blexer *lexer)
-{
-    int ch = lgetc(lexer);
+static void save_char(blexer *lexer, int ch) {
     struct blexerbuf *buf = &lexer->buf;
     if (buf->len >= buf->size) {
         size_t size = buf->size << 1;
         buf->s = be_realloc(lexer->vm, buf->s, buf->size, size);
         buf->size = size;
     }
     buf->s[buf->len++] = (char)ch;
+}
+
+/* save and next */
+static int save(blexer *lexer)
+{
+    int ch = lgetc(lexer);
+    save_char(lexer, ch);
     return next(lexer);
 }
 
@@ -247,7 +253,7 @@ static int skip_newline(blexer *lexer)
         next(lexer); /* skip "\n\r" or "\r\n" */
     }
     lexer->linenumber++;
-    return lexer->cursor;
+    return lexer->reader.cursor;
 }
 
 static void skip_comment(blexer *lexer)
@@ -373,12 +379,217 @@ static btokentype scan_numeral(blexer *lexer)
     return type;
 }
 
+/* structure for a temporary reader used by transpiler, with attributes for an allocated buffer and size */
+struct blexerreader_save {
+    struct blexerreader reader;    
+    char* s;    
+    size_t size;
+    char cursor;
+};
+
+/* buf reader for transpiled code from f-strings */
+/* it restores the original reader when the transpiler buffer is empty */
+/* the first pass returns a single byte buffer with the saved cursor */
+/* the second pass restores the original reader */
+static const char* _bufgets(struct blexer* lexer, void *data, size_t *size)
+{
+    /* this is called once the temporaty transpiler buffer is empty */
+    struct blexerreader *reader = &lexer->reader;       /* current reader which is temporary only for the transpiler */
+    struct blexerreader_save *reader_save = data;          /* original reader that needs to be restored once the buffer is empty */
+
+    /* first case, we saved the cursor (fist char), we server it now */
+    if (reader_save->reader.cursor >= 0) {                        /* serve the previously saved cursor */
+        /* copy cursor to a 'char' type */
+        reader_save->cursor = reader_save->reader.cursor;
+        reader_save->reader.cursor = -1;                          /* no more cursor saved */
+        *size = 1;
+        return &reader_save->cursor;
+    }
+
+    /* second case, the saved cursor was returned, now restore the normal flow of the original reader */
+    /* restore the original reader */
+    *reader = reader_save->reader;
+
+    /* free the memory from the structure */
+    be_free(lexer->vm, reader_save->s, reader_save->size);                  /* free the buffer */
+    be_free(lexer->vm, reader_save, sizeof(struct blexerreader_save));      /* free the structure */
+
+    if (!reader->len) {     /* just in case the original buffer was also */
+        return reader->readf(lexer, reader->data, size);
+    }
+    /* the following is not necessary, but safer */
+    *size = reader->len;
+    return reader->s;
+}
+
+static btokentype scan_string(blexer *lexer);   /* forward declaration */
+
+/* scan f-string and transpile it to `format(...)` syntax then feeding the normal lexer and parser */
+static void scan_f_string(blexer *lexer)
+{
+    char ch;
+    clear_buf(lexer);
+    scan_string(lexer);         /* first scan the entire string in lexer->buf */
+
+    /* save original reader until the transpiled is processed */
+    /* reader will be restored by the reader function once the transpiled buffer is empty */
+    struct blexerreader_save *reader_save = (struct blexerreader_save *) be_malloc(lexer->vm, sizeof(struct blexerreader_save));           /* temporary reader */
+    reader_save->reader = lexer->reader;
+
+    /* save blexerbuf which contains the unparsed_fstring */
+    struct blexerbuf buf_unparsed_fstr = lexer->buf;
+
+    /* prepare and allocated a temporary buffer to save parsed f_string */
+    lexer->buf.size = lexer->buf.size + 20;
+    lexer->buf.s = be_malloc(lexer->vm, lexer->buf.size);
+    lexer->buf.len = 0;
+
+    /* parse f_string */
+    /* First pass, check syntax and extract string literals, and format */
+    save_char(lexer, '(');
+    save_char(lexer, '"');
+    for (size_t i = 0; i < buf_unparsed_fstr.len; i++) {
+        ch = buf_unparsed_fstr.s[i];
+        switch (ch) {
+            case '%':       /* % needs to be encoded as %% */
+                save_char(lexer, '%');
+                save_char(lexer, '%');
+                break;
+            case '\\':       /* \ needs to be encoded as \\ */
+                save_char(lexer, '\\');
+                save_char(lexer, '\\');
+                break;
+            case '"':       /* " needs to be encoded as \" */
+                save_char(lexer, '\\');
+                save_char(lexer, '"');
+                break;
+            case '}':       /* }} is converted as } yet we tolerate a single } */
+                if ((i+1 < buf_unparsed_fstr.len) && (buf_unparsed_fstr.s[i+1] == '}')) { i++; }      /* if '}}' replace with '}' */
+                save_char(lexer, '}');
+                break;
+            default:        /* copy any other character */
+                save_char(lexer, ch);
+                break;
+            case '{':       /* special case for { */
+                i++;        /* in all cases skip to next char */
+                if ((i < buf_unparsed_fstr.len) && (buf_unparsed_fstr.s[i] == '{')) {
+                    save_char(lexer, '{');      /* {{ is simply encoded as { and continue parsing */
+                } else {
+                    /* we still don't know if '=' is present, so we copy the expression each time, and rollback if we find out the '=' is not present */
+                    size_t rollback = lexer->buf.len;       /* mark the end of string for later rollback if '=' is not present */
+                    /* parse inner part */
+                    /* skip everything until either ':' or '}' or '=' */
+                    /* if end of string is reached before '}' raise en error */
+                    for (; i < buf_unparsed_fstr.len; i++) {
+                        ch = buf_unparsed_fstr.s[i];
+                        if (ch == ':' || ch == '}') { break; }
+                        save_char(lexer, ch);       /* copy any character unless it's ':' or '}' */
+                        if (ch == '=') { break; }   /* '=' is copied but breaks parsing as well */
+                    }
+                    /* safe check if we reached the end of the string */
+                    if (i >= buf_unparsed_fstr.len) { be_raise(lexer->vm, "syntax_error", "'}' expected"); }
+                    /* if '=' detected then do some additional checks */
+                    if (ch == '=') {
+                        i++;        /* skip '=' and check we haven't reached the end */
+                        if (i >= buf_unparsed_fstr.len) { be_raise(lexer->vm, "syntax_error", "'}' expected"); }
+                        ch = buf_unparsed_fstr.s[i];
+                        if ((ch != ':') && (ch != '}')) {   /* '=' must be immediately followed by ':' or '}' */
+                            be_raise(lexer->vm, "syntax_error", "':' or '}' expected after '='");
+                        }
+                    } else {
+                        /* no '=' present, rollback the text of the expression */
+                        lexer->buf.len = rollback;
+                    }
+                    save_char(lexer, '%');       /* start format encoding */
+                    if (ch == ':') {
+                        /* copy format */
+                        i++;
+                        if ((i < buf_unparsed_fstr.len) && (buf_unparsed_fstr.s[i] == '%')) { i++; }      /* skip '%' following ':' */
+                        for (; i < buf_unparsed_fstr.len; i++) {
+                            ch = buf_unparsed_fstr.s[i];
+                            if (ch == '}') { break; }
+                            save_char(lexer, ch);
+                        }
+                        if (i >= buf_unparsed_fstr.len) { be_raise(lexer->vm, "syntax_error", "'}' expected"); }
+                    } else {
+                        /* if no formatting, output '%s' */
+                        save_char(lexer, 's');
+                    }
+                }
+                break;
+        }
+    }
+    save_char(lexer, '"');      /* finish format string */
+
+    /* Second pass - add arguments if any */
+    for (size_t i = 0; i < buf_unparsed_fstr.len; i++) {
+        /* skip any character that is not '{' followed by '{' */
+        if (buf_unparsed_fstr.s[i] == '{') {
+            i++;        /* in all cases skip to next char */
+            if ((i < buf_unparsed_fstr.len) && (buf_unparsed_fstr.s[i] == '{')) { continue; }
+            /* extract argument */
+            save_char(lexer, ',');       /* add ',' to start next argument to `format()` */
+            for (; i < buf_unparsed_fstr.len; i++) {
+                ch = buf_unparsed_fstr.s[i];
+                if (ch == '=' || ch == ':' || ch == '}') { break; }
+                save_char(lexer, ch);   /* copy expression until we reach ':', '=' or '}' */
+            }
+            /* no need to check for end of string here, it was done already in first pass */
+            if (ch == ':' || ch == '=') {       /* if '=' or ':', skip everyting until '}' */
+                i++;
+                for (; i < buf_unparsed_fstr.len; i++) {
+                    ch = buf_unparsed_fstr.s[i];
+                    if (ch == '}') { break; }
+                }
+            }
+        }
+    }
+    save_char(lexer, ')');      /* add final ')' */
+
+    /* Situation now: */
+    /* `buf_unparsed_fstr` contains the buffer of the input unparsed f-string, ex: "age: {age:2i}" */
+    /* `lexer->buf` contains the buffer of the transpiled f-string without call to format(), ex: '("age: %2i", age)' */
+    /* `reader_save` contains the original reader to continue parsing after f-string */
+    /* `lexer->reader` will contain a temporary reader from the parsed f-string */
+
+    /* extract the parsed f-string from the temporary buffer (needs later deallocation) */
+    char * parsed_fstr_s = lexer->buf.s;        /* needs later deallocation with parsed_fstr_size */
+    size_t parsed_fstr_len = lexer->buf.len;
+    size_t parsed_fstr_size = lexer->buf.size;
+
+    /* restore buf to lexer */
+    lexer->buf = buf_unparsed_fstr;
+
+    /* change the temporary reader to the parsed f-string */
+    lexer->reader.len = parsed_fstr_len;
+    lexer->reader.data = (void*) reader_save;      /* link to the saved context */
+    lexer->reader.s = parsed_fstr_s;     /* reader is responisble to deallocate later this buffer */
+    lexer->reader.readf = _bufgets;
+
+    /* add information needed for `_bufgets` to later deallocate the buffer */
+    reader_save->size = parsed_fstr_size;
+    reader_save->s = parsed_fstr_s;
+
+    /* start parsing the parsed f-string, which is btw always '(' */
+    next(lexer);
+
+    /* remember that we are still in `scan_identifier()`, we replace the 'f' identifier to 'format' which is the global function to call */
+    static const char FORMAT[] = "format";
+    lexer->buf.len = sizeof(FORMAT) - 1;        /* we now that buf size is at least SHORT_STR_LEN (32) */
+    memmove(lexer->buf.s, FORMAT, lexer->buf.len);
+}
+
 static btokentype scan_identifier(blexer *lexer)
 {
     int type;
     bstring *s;
     save(lexer);
     match(lexer, is_word);
+    /* check if the form is f"aaaa" or f'aaa' */
+    char ch = lgetc(lexer);
+    if ((lexer->buf.len == 1) && (lexer->buf.s[0] == 'f') && (ch == '"' || ch == '\'')) {
+        scan_f_string(lexer);
+    }
     s = buf_tostr(lexer);
     type = str_extra(s);
     if (type >= KeyIf && type < type_count()) {

lib/libesp32/berry/src/be_lexer.h

@@ -97,6 +97,7 @@ struct blexerreader {
     size_t len;
     void *data;
     breader readf;
+    int cursor;
 };
 
 struct blexerbuf {
@@ -123,7 +124,6 @@ typedef struct blexer {
     struct blexerreader reader;
     bmap *strtab;
     bvm *vm;
-    int cursor;
 } blexer;
 
 void be_lexer_init(blexer *lexer, bvm *vm,

lib/libesp32/berry/src/be_object.h

@@ -195,7 +195,8 @@ typedef struct {
     bntvfunc destroy;
 } bcommomobj;
 
-typedef const char* (*breader)(void*, size_t*);
+struct blexer;
+typedef const char* (*breader)(struct blexer*, void*, size_t*);
 
 #define cast(_T, _v)            ((_T)(_v))
 #define cast_int(_v)            cast(int, _v)