upd

qic.c

@@ -351,8 +351,13 @@ typedef struct {
     T_NUMBER,
     T_STRING,
     T_FSTRING,
+    T_USTRING,
     T_NAME,
 
+    T_TRUE,
+    T_FALSE,
+    T_NIL,
+
     T_VAR,
     T_LET,
     T_CONST,
@@ -675,6 +680,8 @@ token_t *tokenize_string(char *source, size_t *pos) {
   return token(T_STRING, buffer_read(text));
 }
 
+int NEEDS_UTF8 = 0;
+
 token_t *next_token(char *source, size_t *pos) {
   if (!source[*pos])
     return token(T_EOF, NULL);
@@ -689,6 +696,17 @@ token_t *next_token(char *source, size_t *pos) {
     token_t *t = tokenize_string(source, pos);
     t->tag = T_FSTRING;
 
+    return t;
+  } else if (source[*pos] == 'u' &&
+            (source[(*pos) + 1] == '"' || source[(*pos) + 1] == '\'' ||
+             source[(*pos) + 1] == '`')) {
+    (*pos)++;
+
+    token_t *t = tokenize_string(source, pos);
+    t->tag = T_USTRING;
+
+    NEEDS_UTF8 = 1;
+
     return t;
   } else if (source[*pos] == '0' &&
              (source[(*pos) + 1] == 'x' || source[(*pos) + 1] == 'b' ||
@@ -842,6 +860,12 @@ token_t *next_token(char *source, size_t *pos) {
       return TK(INCLUDE);
     else if (strcmp(name, "macro") == 0)
       return TK(MACRO);
+    else if (strcmp(name, "true") == 0)
+      return TK(TRUE);
+    else if (strcmp(name, "false") == 0)
+      return TK(FALSE);
+    else if (strcmp(name, "nil") == 0)
+      return TK(NIL);
 
     return token(T_NAME, name);
   } else if (strncmp(&source[*pos], "==", 2) == 0 && ++(*pos) && ++(*pos))
@@ -1549,7 +1573,7 @@ node_t *parse_primary(list_t *tokens, size_t *pos) {
     EXPECT(RCB, "}");
 
     return NODEH(TABLE, table);
-  } else if (MATCH(NUMBER) || MATCH(STRING) || MATCH(FSTRING) || MATCH(NAME))
+  } else if (MATCH(NUMBER) || MATCH(STRING) || MATCH(FSTRING) || MATCH(USTRING) || MATCH(NAME) || MATCH(TRUE) || MATCH(FALSE) || MATCH(NIL))
     return NODET(LITERAL, tokens->data[(*pos) - 1]);
 
   if (MATCH(RPAR)) {
@@ -2631,7 +2655,7 @@ node_t *parse_mprimary(list_t *tokens, size_t *pos) {
     EXPECT(RPAR, ")");
 
     return a;
-  } else if (MATCH(NAME)) {
+  } else if (MATCH(NAME) || MATCH(TRUE) || MATCH(FALSE) || MATCH(NIL)) {
     token_t *t = tokens->data[(*pos) - 1];
 
     int macro = 0;
@@ -3493,6 +3517,45 @@ const char *STD[][2] = {
              "    return bytes([0xf0 | ((c >> 18) & 0x7), 0x80 | ((c >> 12) & "
              "0x3f), 0x80 | ((c >> 6) & 0x3f), 0x80 | (c & 0x3f)])\n"
              "  throw \"malformed codepoint\"\n"
+             "}\n"
+             "class ustr {\n"
+             "  _ucs = nil\n"
+             "  constructor (this, s=\"\") {\n"
+             "    if type(s) == \"list\" && all(map(func (x): type(x) == \"number\", s)) {\n"
+             "      this._ucs = s\n"
+             "      return\n"
+             "    }\n"
+             "    s = str(s)\n"
+             "    this._ucs = []\n"
+             "    for var i = 0; i < len(s) {\n"
+             "      let [c, l] = utf8_decode(slice(s, i))\n"
+             "      this._ucs.push(c)\n"
+             "      i += l\n"
+             "    }\n"
+             "  }\n"
+             "  nth (this, i): str(utf8_encode(this._ucs[i]))\n"
+             "  __type (this): \"ustr\"\n"
+             "  __str (this) {\n"
+             "    var s = \"\"\n"
+             "    for var uc of this._ucs\n"
+             "      s += str(utf8_encode(uc))\n"
+             "    return s\n"
+             "  }\n"
+             "  __len (this): len(this._ucs)\n"
+             "  __index (this, i): this._ucs[i]\n"
+             "  __index_set (this): throw\n"
+             "  __iter (this): this._ucs\n"
+             "  __list (this): list_copy(this._ucs)\n"
+             "  __add (this, other) {\n"
+             "    if type(other) != \"ustr\"\n"
+             "      throw\n"
+             "    return ustr(this._ucs + other._ucs)\n"
+             "  }\n"
+             "  __equals (this, other) {\n"
+             "    if type(other) != \"ustr\"\n"
+             "      return false\n"
+             "    return this._ucs == other._ucs\n"
+             "  }\n"
              "}\n"},
 
     {"thread",
@@ -4911,6 +4974,10 @@ void compile_node(buffer_t *gbuf, buffer_t *buf, list_t *ctx, table_t *ltab,
       EMIT("__%s%d(state)", PREFIX, gid);
     } break;
 
+    case T_USTRING:      
+      EMIT("qi_call(state, qi_get(state, \"ustr\"), qi_list_push(qi_list_make(), qi_make_string(state, \"%s\")))", node->t->text);
+      break;
+
     case T_NAME: {
       char *name = node->t->text;
       node_t *n = const_get(name);
@@ -4921,6 +4988,10 @@ void compile_node(buffer_t *gbuf, buffer_t *buf, list_t *ctx, table_t *ltab,
         EMIT("qi_get(state, \"%s\")", name);
     } break;
 
+    case T_TRUE: EMIT("state->_true"); break;
+    case T_FALSE: EMIT("state->_false"); break;
+    case T_NIL: EMIT("state->nil"); break;
+
     default:
       COMPILE_ERROR("not yet implemented");
     }
@@ -5376,6 +5447,9 @@ void compile_node(buffer_t *gbuf, buffer_t *buf, list_t *ctx, table_t *ltab,
       EMIT(")) {\n");
     }
 
+    CTXPUSH("scope");
+    EMIT("qi_new_scope(state);\n");
+
     LPUSH(gid);
     CTXPUSH("for");
     compile_node(gbuf, buf, ctx, ltab, lstk, sstk, lbl, node->d);
@@ -5391,9 +5465,15 @@ void compile_node(buffer_t *gbuf, buffer_t *buf, list_t *ctx, table_t *ltab,
     CTXPOP();
     LPOP();
 
+    CTXPOP();
+    EMIT("qi_old_scope(state);\n");
+
     EMIT("}\n");
 
+    EMIT("goto __exit%d;\n", gid);
     EMIT("__break%d:;\n", gid);
+    EMIT("qi_old_scope(state);\n");
+    EMIT("__exit%d:;\n", gid);
 
     CTXPOP();
     EMIT("qi_old_scope(state);\n");
@@ -5447,6 +5527,9 @@ void compile_node(buffer_t *gbuf, buffer_t *buf, list_t *ctx, table_t *ltab,
            "qi_make_number(state, i)));\n",
            node->t->text, varname);
 
+    CTXPUSH("scope");
+    EMIT("qi_new_scope(state);\n");
+
     LPUSH(gid);
     CTXPUSH("for");
     compile_node(gbuf, buf, ctx, ltab, lstk, sstk, lbl, node->b);
@@ -5454,9 +5537,16 @@ void compile_node(buffer_t *gbuf, buffer_t *buf, list_t *ctx, table_t *ltab,
     LPOP();
 
     EMIT("__continue%d:;\n", gid);
+
+    CTXPOP();
+    EMIT("qi_old_scope(state);\n");
+
     EMIT("}\n");
 
+    EMIT("goto __exit%d;\n", gid);
     EMIT("__break%d:;\n", gid);
+    EMIT("qi_old_scope(state);\n");
+    EMIT("__exit%d:;\n", gid);
 
     CTXPOP();
     EMIT("qi_old_scope(state);\n");
@@ -5831,6 +5921,11 @@ void compile_into(char *source, buffer_t *gbuf, buffer_t *buf, list_t *ctx,
                   table_t *ltab, int_stack_t *lstk, int_stack_t *sstk,
                   list_t *lbl) {
   node_t *n = parse(source);
+  if (NEEDS_UTF8) {
+    NEEDS_UTF8 = 0;
+
+    require_once(gbuf, buf, ctx, ltab, lstk, sstk, lbl, "utf8");
+  }
 
   compile_node(gbuf, buf, ctx, ltab, lstk, sstk, lbl, n);
 }