Skip to content

Commit

Permalink
Merge pull request #356 from PlutoLang/fix-tokens
Browse files Browse the repository at this point in the history 
Fix luaX_tokens not containing Pluto symbols
  • Loading branch information
@Sainan
Sainan authored Sep 6, 2023
2 parents 0f5b5ad + 4f2e189 commit 1026fe5
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 147 deletions.
64 changes: 15 additions & 49 deletions src/llex.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ static const char *const luaX_tokens [] = {
"return", "then", "true", "until", "while",
"//", "..", "...", "==", ">=", "<=", "~=",
"<<", ">>", "::", "<eof>",
"<number>", "<integer>", "<name>", "<string>"
"<number>", "<integer>", "<name>", "<string>",
"**", "??", ":=",
};


Expand Down Expand Up @@ -540,38 +541,6 @@ static void read_string (LexState *ls, int del, SemInfo *seminfo) {
}
}

/* assigns a reserved augmentation symbol to the lexer state token (ls->lasttoken) */
static int llex_augmented (lua_Integer& i, int c) {
switch (c) {
case '+': {
i = TK_CADD;
return 1;
}
case '*': {
i = TK_CMUL;
return 1;
}
case '%': {
i = TK_CMOD;
return 1;
}
case '^': {
i = TK_CPOW;
return 1;
}
case '|': {
i = TK_CBOR;
return 1;
}
case '&': {
i = TK_CBAND;
return 1;
}
default: {
return 0; /* failure */
}
}
}

static int llex (LexState *ls, SemInfo *seminfo) {
luaZ_resetbuffer(ls->buff);
Expand All @@ -593,7 +562,7 @@ static int llex (LexState *ls, SemInfo *seminfo) {
ls->appendLineBuff('-');
if (check_next1(ls, '=')) { /* compound op */
ls->appendLineBuff('=');
seminfo->i = TK_CSUB;
seminfo->i = '-';
return '=';
}
else {
Expand Down Expand Up @@ -653,7 +622,7 @@ static int llex (LexState *ls, SemInfo *seminfo) {
else if (check_next1(ls, '<')) {
if (check_next1(ls, '=')) { /* compound support */
ls->appendLineBuff("<<=");
seminfo->i = TK_CSHL; /* <<= */
seminfo->i = TK_SHL; /* <<= */
return '=';
}
else {
Expand All @@ -675,7 +644,7 @@ static int llex (LexState *ls, SemInfo *seminfo) {
else if (check_next1(ls, '>')) {
if (check_next1(ls, '=')) { /* compound support */
ls->appendLineBuff(">>=");
seminfo->i = TK_CSHR; /* >>= */
seminfo->i = TK_SHR; /* >>= */
return '=';
}
else {
Expand All @@ -692,7 +661,7 @@ static int llex (LexState *ls, SemInfo *seminfo) {
next(ls);
if (check_next1(ls, '=')) { /* compound support */
ls->appendLineBuff("/=");
seminfo->i = TK_CDIV;
seminfo->i = '/';
return '=';
} else {
if (check_next1(ls, '/')) {
Expand All @@ -702,7 +671,7 @@ static int llex (LexState *ls, SemInfo *seminfo) {
}
else { /* floor division compound support */
ls->appendLineBuff("//=");
seminfo->i = TK_CIDIV;
seminfo->i = TK_IDIV;
return '=';
}
}
Expand Down Expand Up @@ -828,7 +797,7 @@ static int llex (LexState *ls, SemInfo *seminfo) {
else {
if (check_next1(ls, '=')) {
ls->appendLineBuff("..=");
seminfo->i = TK_CCAT;
seminfo->i = TK_CONCAT;
return '=';
} else {
ls->appendLineBuff("..");
Expand Down Expand Up @@ -907,7 +876,7 @@ static int llex (LexState *ls, SemInfo *seminfo) {
next(ls);
if (check_next1(ls, '?')) {
if (check_next1(ls, '=')) {
ls->appendLineBuff("?\?=");
ls->appendLineBuff("??=");
seminfo->i = TK_COAL;
return '=';
} else {
Expand All @@ -924,13 +893,13 @@ static int llex (LexState *ls, SemInfo *seminfo) {
next(ls);
if (check_next1(ls, '=')) {
ls->appendLineBuff("*=");
seminfo->i = TK_CMUL;
seminfo->i = '*';
return '='; /* '*=' */
}
else if (check_next1(ls, '*')) { /* got '**' */
if (check_next1(ls, '=')) { /* compound support; **= */
ls->appendLineBuff("**=");
seminfo->i = TK_CPOW;
seminfo->i = TK_POW;
return '=';
}
else {
Expand All @@ -950,13 +919,10 @@ static int llex (LexState *ls, SemInfo *seminfo) {
int c = ls->current;
next(ls);
if (check_next1(ls, '=')) {
if (llex_augmented(seminfo->i, c) != 1) {
lexerror(ls, "unsupported augmented assignment", c);
} else {
ls->appendLineBuff(c);
ls->appendLineBuff('=');
return '=';
}
seminfo->i = c;
ls->appendLineBuff(c);
ls->appendLineBuff('=');
return '=';
} else {
ls->appendLineBuff(c);
return c;
Expand Down
12 changes: 3 additions & 9 deletions src/llex.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,15 +54,9 @@ enum RESERVED {
TK_EOS, TK_FLT,
TK_INT, TK_NAME, TK_STRING,
/* Pluto symbols */
TK_CSUB, TK_CSHL, /* subtraction & shift left */
TK_CSHR, TK_CBAND, /* shift right & bitwise AND */
TK_CADD, TK_CMUL, /* addition and multiplication */
TK_CMOD, TK_CBOR, /* modulo and bitwise OR */
TK_CBXOR, /* bitwise XOR */
TK_CIDIV, TK_CDIV, /* integer and float division */
TK_CPOW, TK_POW, /* exponents / power */
TK_CCAT, TK_COAL, /* concatenation & null coal. */
TK_WALRUS, /* walrus operator */
TK_POW, /* exponents / power */
TK_COAL, /* null coal. */
TK_WALRUS, /* walrus operator */
};

#define FIRST_COMPAT TK_PUSE
Expand Down
107 changes: 18 additions & 89 deletions src/lparser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3020,77 +3020,6 @@ static void check_conflict (LexState *ls, struct LHS_assign *lh, expdesc *v) {
}
}

/*
gets the supported binary compound operation (if any)
gives OPR_NOBINOPR if the operation does not have compound support.
returns a status (0 false, 1 true) and takes a pointer to set.
this allows for seamless conditional implementation, avoiding a getcompoundop call for every Lua assignment.
*/
static int getcompoundop (lua_Integer i, BinOpr *op) {
switch (i) {
case TK_CCAT: {
*op = OPR_CONCAT;
return 1; /* concatenation */
}
case TK_CADD: {
*op = OPR_ADD; /* addition */
return 1;
}
case TK_CSUB: {
*op = OPR_SUB; /* subtraction */
return 1;
}
case TK_CMUL: {
*op = OPR_MUL; /* multiplication */
return 1;
}
case TK_CMOD: {
*op = OPR_MOD; /* modulo */
return 1;
}
case TK_CDIV: {
*op = OPR_DIV; /* float division */
return 1;
}
case TK_CPOW: {
*op = OPR_POW; /* power */
return 1;
}
case TK_CIDIV: {
*op = OPR_IDIV; /* integer division */
return 1;
}
case TK_CBOR: {
*op = OPR_BOR; /* bitwise OR */
return 1;
}
case TK_CBAND: {
*op = OPR_BAND; /* bitwise AND */
return 1;
}
case TK_CBXOR: {
*op = OPR_BXOR; /* bitwise XOR */
return 1;
}
case TK_CSHL: {
*op = OPR_SHL; /* shift left */
return 1;
}
case TK_CSHR: {
*op = OPR_SHR; /* shift right */
return 1;
}
case TK_COAL: {
*op = OPR_COAL;
return 1;
}
default: {
*op = OPR_NOBINOPR;
return 0;
}
}
}

/*
compound assignment function
determines the binary operation to perform depending on lexer state tokens (ls->lasttoken)
Expand Down Expand Up @@ -3148,29 +3077,29 @@ static void restassign (LexState *ls, struct LHS_assign *lh, int nvars) {
leavelevel(ls);
}
else { /* restassign -> '=' explist */
BinOpr op; /* binary operation from lexer state */
if (getcompoundop(ls->t.seminfo.i, &op) != 0) { /* is there a saved binop? */
check(ls, '=');
BinOpr op = getbinopr((int)ls->t.seminfo.i); /* binary operation from lexer state */
if (op != OPR_NOBINOPR) { /* is there a saved binop? */
check_condition(ls, nvars == 1, "unsupported tuple assignment");
compoundassign(ls, &lh->v, op); /* perform binop & assignment */
return; /* avoid default */
}
else if (testnext(ls, '=')) { /* no requested binop, continue */
TypeHint prop{};
ParserContext ctx = ((nvars == 1) ? PARCTX_CREATE_VAR : PARCTX_CREATE_VARS);
ls->pushContext(ctx);
int nexps = explist(ls, &e, &prop);
ls->popContext(ctx);
if (nexps != nvars)
adjust_assign(ls, nvars, nexps, &e);
else {
luaK_setoneret(ls->fs, &e); /* close last expression */
if (lh->v.k == VLOCAL) { /* assigning to a local variable? */
exp_propagate(ls, e, prop);
process_assign(ls, getlocalvardesc(ls->fs, lh->v.u.var.vidx), prop, line);
}
luaK_storevar(ls->fs, &lh->v, &e);
return; /* avoid default */
luaX_next(ls);
TypeHint prop{};
ParserContext ctx = ((nvars == 1) ? PARCTX_CREATE_VAR : PARCTX_CREATE_VARS);
ls->pushContext(ctx);
int nexps = explist(ls, &e, &prop);
ls->popContext(ctx);
if (nexps != nvars)
adjust_assign(ls, nvars, nexps, &e);
else {
luaK_setoneret(ls->fs, &e); /* close last expression */
if (lh->v.k == VLOCAL) { /* assigning to a local variable? */
exp_propagate(ls, e, prop);
process_assign(ls, getlocalvardesc(ls->fs, lh->v.u.var.vidx), prop, line);
}
luaK_storevar(ls->fs, &lh->v, &e);
return; /* avoid default */
}
}
init_exp(&e, VNONRELOC, ls->fs->freereg-1); /* default assignment */
Expand Down

0 comments on commit 1026fe5

Please sign in to comment.