lingcoder

# syntax.ling - per-line tokenizers for Ling, TOML, Markdown and friends.
# A token list is flat: [start, length, colour, start, length, colour, ...] in char units.
# Ling has no block comments and no multi-line strings, so every line tokenizes on its own.
# Keyword tables come from editors/vscode-ling/syntaxes/ling.tmLanguage.json (the IDE's
# colours follow its Ling Dark theme).

# ── keyword tables ────────────────────────────────────────────────────────────
# space-delimited words (ASCII and Cyrillic: they sit between spaces / punctuation)
fn syn_kw_words() {
    return "|if|else|while|for|in|match|return|post|give|again|stop|try|choose|can|change|where|fit|form|do|spawn|wait|as|start|result|sure|maybe|bind|fn|mod|type|use|own|lend|share|move|copy|pure|async|func|module|fonction|lier|faire|si|sinon|tant_que|pour|dans|correspondre|retourner|posséder|prêter|partager|déplacer|copier|asynchrone|attendre|comme|où|publier|donner|convenir|forme|choisir|changer|arrêter|encore|essayer|sûr|peut_être|pur|engendrer|démarrer|résultat|peut|funktion|binden|machen|modul|typ|wenn|sonst|solange|für|abgleichen|zurück|besitzen|leihen|teilen|bewegen|kopieren|asynchron|warten|als|wobei|veröffentlichen|geben|passen|wählen|kann|ändern|stoppen|wieder|versuchen|sicher|vielleicht|rein|erzeugen|ergebnis|nutzen|utiliser|связать|сделать|если|иначе|пока|для|вернуть|сопоставить|модуль|тип|как|создать|ждать|асинхронно|пробовать|владеть|одолжить|делиться|переместить|копировать|начать|стоп|снова|выбрать|может|изменить|форма|уверен|возможно|результат|опубликовать|дать|в|функция|"
}
fn syn_const_words() {
    return "|true|false|ok|bad|none|err|vrai|faux|wahr|falsch|bon|mauvais|rien|gut|schlecht|nichts|"
}
fn syn_type_words() {
    return "|number|text|bool|list|map|tuple|"
}
# CJK / Thai / Arabic / Hebrew / Devanagari keyword forms, matched by prefix at a word
# start, longest first within each entry. Kept as one big pipe table.
fn syn_kw_cjk() {
    return "|函数|関数|モジュール|바인드|束縛|实行|実行|함수|모듈|使う|ฟังก์ชัน|否则|共享|移动|复制|异步|等待|形式|选择|改变|停止|继续|尝试|确定|可能|生成|拥有|止まれ|戻る|帰る|試す|待つ|非同期|起動|一致|の中|ために|一方|아니면|기다려|비동기|반환|귀환|만약|조건|동안|반복|위해|안에|매치|시도|생성|멈춤|계속|실행|もし|執行|灵符|ผูก|ทำอีก|มิฉะนั้น|ขณะที่|สำหรับ|จับคู่|ไม่พร้อมกัน|เป็นเจ้าของ|ให้ยืม|เริ่ม|ส่ง|รูปแบบ|เลือก|สามารถ|แบ่งปัน|เปลี่ยนแปลง|ย้าย|คัดลอก|หยุด|ลอง|แน่นอน|อาจจะ|ผลลัพธ์|บริสุทธิ์|อะซิงค์|สร้าง|ตรงกับ|ชนิด|โมดูล|ถ้า|ใน|คืน|รอ|ให้|ทำ|บาँधो|करो|अगर|नहींतो|जबकि|केलिए|वापस|ربط|افعل|إذا|وإلا|بينما|لأجل|في|أعد|دالة|استخدم|وحدة|نوع|令|函|核|符|载|引|执|做|若|如|当|历|于|配|归|借|移|复|异|待|为|出|给|予|选|能|变|止|试|确|或|启|間|繰|他|墨|数|文|布|列|图|元|真|假|偽|참|거짓|จริง|เท็จ|好|可|坏|误|无|"
}

# one lookup string per kind, cached in the store once
fn syn_init() {
    if st_has("syn_ready") > 0.5 { return 0.0 }
    st_set("syn_kw", syn_kw_words())
    st_set("syn_const", syn_const_words())
    st_set("syn_type", syn_type_words())
    st_set("syn_cjk", syn_kw_cjk())
    st_set("syn_ready", 1.0)
    return 0.0
}

# language of a file by extension
fn syn_lang(path) {
    bind e = p_ext(path)
    if e == "ling" || e == "灵" || e == "령" || e == "霊" || e == "ลิง" { return "ling" }
    if e == "toml" { return "toml" }
    if e == "md" { return "md" }
    if e == "lscene" || e == "lcut" { return "rec" }
    if e == "json" { return "json" }
    if e == "ignore" || p_base(path) == "ling.ignore" || p_base(path) == ".gitignore" { return "cfg" }
    if e == "txt" || e == "log" || e == "" { return "cfg" }
    if e == "bat" || e == "cmd" || e == "sh" { return "sh" }
    return "plain"
}

# is this char part of a word? (not an operator / space / quote)
fn syn_is_word(ch) {
    if ch == "" || ch == " " || ch == "\t" { return false }
    return str_find("()[]{}+-*/%=<>!&|,.:;\"'#@^~?$`", ch) < 0.0
}
fn syn_is_digit(ch) {
    if ch == "" { return false }
    return str_find("0123456789", ch) >= 0.0
}

# ── Ling tokenizer ────────────────────────────────────────────────────────────
fn syn_ling(ln) {
    syn_init()
    bind out = []
    bind n = str_len(ln)
    bind kw = st_get("syn_kw", "|")
    bind kconst = st_get("syn_const", "|")
    bind ktype = st_get("syn_type", "|")
    bind kcjk = st_get("syn_cjk", "|")
    bind i = 0.0
    while i < n {
        bind ch = str_slice(ln, i, 1.0)
        # comments: # or //
        if ch == "#" {
            bind out = list_push(out, i)  bind out = list_push(out, n - i)  bind out = list_push(out, E_COMMENT)
            bind i = n
        } else if ch == "/" && i + 1.0 < n && str_slice(ln, i + 1.0, 1.0) == "/" {
            bind out = list_push(out, i)  bind out = list_push(out, n - i)  bind out = list_push(out, E_COMMENT)
            bind i = n
        } else if ch == "\"" {
            # string with \" escapes
            bind j = i + 1.0
            bind run = true
            while j < n && run {
                bind c2 = str_slice(ln, j, 1.0)
                if c2 == "\\" { bind j = j + 2.0 }
                else if c2 == "\"" { bind run = false  bind j = j + 1.0 }
                else { bind j = j + 1.0 }
            }
            bind out = list_push(out, i)  bind out = list_push(out, j - i)  bind out = list_push(out, E_STRING)
            bind i = j
        } else if syn_is_digit(ch) {
            bind j = i + 1.0
            while j < n && (syn_is_digit(str_slice(ln, j, 1.0)) || str_slice(ln, j, 1.0) == ".") { bind j = j + 1.0 }
            bind out = list_push(out, i)  bind out = list_push(out, j - i)  bind out = list_push(out, E_NUMBER)
            bind i = j
        } else if syn_is_word(ch) {
            # a word: ASCII / unicode letters, digits, _
            bind j = i + 1.0
            while j < n && syn_is_word(str_slice(ln, j, 1.0)) { bind j = j + 1.0 }
            bind w = str_slice(ln, i, j - i)
            bind col = E_TEXT
            bind wl = 0.0 - 1.0
            if str_find(kw, "|" + w + "|") >= 0.0 { bind col = E_KEYWORD }
            else if str_find(kconst, "|" + w + "|") >= 0.0 { bind col = E_CONST }
            else if str_find(ktype, "|" + w + "|") >= 0.0 { bind col = E_TYPE }
            else if str_find(kcjk, "|" + w + "|") >= 0.0 { bind col = E_KEYWORD }
            else {
                # CJK run: try keyword prefixes (4..1 chars) at the start of the run
                bind pl = 4.0
                while pl >= 1.0 && wl < 0.0 {
                    if pl <= j - i {
                        if str_find(kcjk, "|" + str_slice(ln, i, pl) + "|") >= 0.0 { bind wl = pl }
                    }
                    bind pl = pl - 1.0
                }
                if wl > 0.0 {
                    bind out = list_push(out, i)  bind out = list_push(out, wl)  bind out = list_push(out, E_KEYWORD)
                    bind i = i + wl
                }
                if wl < 0.0 {
                    # call? next non-space char is (
                    bind k = j
                    while k < n && str_slice(ln, k, 1.0) == " " { bind k = k + 1.0 }
                    if k < n && str_slice(ln, k, 1.0) == "(" { bind col = E_FUNC }
                }
            }
            if wl < 0.0 {
                bind out = list_push(out, i)  bind out = list_push(out, j - i)  bind out = list_push(out, col)
                bind i = j
            }
        } else if ch == " " || ch == "\t" {
            bind i = i + 1.0
        } else {
            # operator / punctuation run of one char
            bind out = list_push(out, i)  bind out = list_push(out, 1.0)  bind out = list_push(out, E_OPER)
            bind i = i + 1.0
        }
    }
    return out
}

# ── TOML tokenizer ────────────────────────────────────────────────────────────
fn syn_toml(ln) {
    bind out = []
    bind n = str_len(ln)
    bind t = str_trim(ln)
    if str_starts_with(t, "#") {
        bind out = list_push(out, 0.0)  bind out = list_push(out, n)  bind out = list_push(out, E_COMMENT)
        return out
    }
    if str_starts_with(t, "[") {
        bind out = list_push(out, 0.0)  bind out = list_push(out, n)  bind out = list_push(out, E_KEYWORD)
        return out
    }
    bind eq = str_find(ln, "=")
    if eq < 0.0 {
        if t != "" { bind out = list_push(out, 0.0)  bind out = list_push(out, n)  bind out = list_push(out, E_TEXT) }
        return out
    }
    # key = value  (strings cyan, numbers blue, booleans blue, comments after value)
    bind out = list_push(out, 0.0)  bind out = list_push(out, eq)  bind out = list_push(out, E_FUNC)
    bind out = list_push(out, eq)  bind out = list_push(out, 1.0)  bind out = list_push(out, E_OPER)
    bind i = eq + 1.0
    while i < n {
        bind ch = str_slice(ln, i, 1.0)
        if ch == "#" {
            bind out = list_push(out, i)  bind out = list_push(out, n - i)  bind out = list_push(out, E_COMMENT)
            bind i = n
        } else if ch == "\"" {
            bind j = i + 1.0
            bind run = true
            while j < n && run {
                bind c2 = str_slice(ln, j, 1.0)
                if c2 == "\\" { bind j = j + 2.0 }
                else if c2 == "\"" { bind run = false  bind j = j + 1.0 }
                else { bind j = j + 1.0 }
            }
            bind out = list_push(out, i)  bind out = list_push(out, j - i)  bind out = list_push(out, E_STRING)
            bind i = j
        } else if syn_is_digit(ch) {
            bind j = i + 1.0
            while j < n && (syn_is_digit(str_slice(ln, j, 1.0)) || str_slice(ln, j, 1.0) == "." || str_slice(ln, j, 1.0) == "-") { bind j = j + 1.0 }
            bind out = list_push(out, i)  bind out = list_push(out, j - i)  bind out = list_push(out, E_NUMBER)
            bind i = j
        } else if syn_is_word(ch) {
            bind j = i + 1.0
            while j < n && syn_is_word(str_slice(ln, j, 1.0)) { bind j = j + 1.0 }
            bind w = str_slice(ln, i, j - i)
            bind col = E_TEXT
            if w == "true" || w == "false" { bind col = E_CONST }
            bind out = list_push(out, i)  bind out = list_push(out, j - i)  bind out = list_push(out, col)
            bind i = j
        } else if ch == " " || ch == "\t" {
            bind i = i + 1.0
        } else {
            bind out = list_push(out, i)  bind out = list_push(out, 1.0)  bind out = list_push(out, E_OPER)
            bind i = i + 1.0
        }
    }
    return out
}

# ── Markdown (light) ──────────────────────────────────────────────────────────
fn syn_md(ln) {
    bind out = []
    bind n = str_len(ln)
    bind t = str_trim(ln)
    if str_starts_with(t, "#") {
        bind out = list_push(out, 0.0)  bind out = list_push(out, n)  bind out = list_push(out, E_KEYWORD)
        return out
    }
    if str_starts_with(t, ">") {
        bind out = list_push(out, 0.0)  bind out = list_push(out, n)  bind out = list_push(out, E_COMMENT)
        return out
    }
    if str_starts_with(t, "- ") || str_starts_with(t, "* ") || str_starts_with(t, "+ ") {
        bind at = str_find(ln, str_slice(t, 0.0, 1.0))
        bind out = list_push(out, at)  bind out = list_push(out, 1.0)  bind out = list_push(out, E_FUNC)
        bind out = list_push(out, at + 1.0)  bind out = list_push(out, n - at - 1.0)  bind out = list_push(out, E_TEXT)
        return out
    }
    if str_starts_with(t, "```") {
        bind out = list_push(out, 0.0)  bind out = list_push(out, n)  bind out = list_push(out, E_STRING)
        return out
    }
    # inline `code` spans cyan, the rest default
    bind i = 0.0
    bind seg = 0.0
    while i < n {
        if str_slice(ln, i, 1.0) == "`" {
            bind j = i + 1.0
            while j < n && str_slice(ln, j, 1.0) != "`" { bind j = j + 1.0 }
            if j < n {
                if i > seg { bind out = list_push(out, seg)  bind out = list_push(out, i - seg)  bind out = list_push(out, E_TEXT) }
                bind out = list_push(out, i)  bind out = list_push(out, j - i + 1.0)  bind out = list_push(out, E_STRING)
                bind i = j + 1.0
                bind seg = i
            } else { bind i = n }
        } else { bind i = i + 1.0 }
    }
    if n > seg { bind out = list_push(out, seg)  bind out = list_push(out, n - seg)  bind out = list_push(out, E_TEXT) }
    return out
}

# ── record files (.lscene / .lcut), config lines, shell ───────────────────────
fn syn_rec(ln) {
    bind out = []
    bind n = str_len(ln)
    if n < 0.5 { return out }
    bind sp = str_find(ln, "|")
    if sp < 0.0 { bind sp = n }
    bind out = list_push(out, 0.0)  bind out = list_push(out, sp)  bind out = list_push(out, E_KEYWORD)
    bind i = sp
    while i < n {
        bind j = str_find(str_slice(ln, i + 1.0, n - i), "|")
        bind seg = n - i
        if j >= 0.0 { bind seg = j + 1.0 }
        bind eq = str_find(str_slice(ln, i, seg), "=")
        if eq >= 0.0 {
            bind out = list_push(out, i)  bind out = list_push(out, eq)  bind out = list_push(out, E_FUNC)
            bind out = list_push(out, i + eq)  bind out = list_push(out, seg - eq)  bind out = list_push(out, E_TEXT)
        } else {
            bind out = list_push(out, i)  bind out = list_push(out, seg)  bind out = list_push(out, E_TEXT)
        }
        bind i = i + seg
    }
    return out
}
fn syn_cfg(ln) {
    bind out = []
    bind n = str_len(ln)
    bind t = str_trim(ln)
    if str_starts_with(t, "#") || str_starts_with(t, ";") {
        bind out = list_push(out, 0.0)  bind out = list_push(out, n)  bind out = list_push(out, E_COMMENT)
        return out
    }
    bind eq = str_find(ln, "=")
    if eq >= 0.0 {
        bind out = list_push(out, 0.0)  bind out = list_push(out, eq)  bind out = list_push(out, E_FUNC)
        bind out = list_push(out, eq)  bind out = list_push(out, n - eq)  bind out = list_push(out, E_TEXT)
        return out
    }
    if n > 0.5 { bind out = list_push(out, 0.0)  bind out = list_push(out, n)  bind out = list_push(out, E_TEXT) }
    return out
}
fn syn_sh(ln) {
    bind out = []
    bind n = str_len(ln)
    bind t = str_trim(ln)
    if str_starts_with(t, "#") || str_starts_with(str_lower(t), "rem ") || str_lower(t) == "rem" {
        bind out = list_push(out, 0.0)  bind out = list_push(out, n)  bind out = list_push(out, E_COMMENT)
        return out
    }
    return syn_ling(ln)
}

# JSON: reuse the toml value scanner on the whole line
fn syn_json(ln) {
    bind out = []
    bind n = str_len(ln)
    bind i = 0.0
    while i < n {
        bind ch = str_slice(ln, i, 1.0)
        if ch == "\"" {
            bind j = i + 1.0
            bind run = true
            while j < n && run {
                bind c2 = str_slice(ln, j, 1.0)
                if c2 == "\\" { bind j = j + 2.0 }
                else if c2 == "\"" { bind run = false  bind j = j + 1.0 }
                else { bind j = j + 1.0 }
            }
            # a key (followed by :) is red, a value cyan
            bind k = j
            while k < n && str_slice(ln, k, 1.0) == " " { bind k = k + 1.0 }
            bind col = E_STRING
            if k < n && str_slice(ln, k, 1.0) == ":" { bind col = E_FUNC }
            bind out = list_push(out, i)  bind out = list_push(out, j - i)  bind out = list_push(out, col)
            bind i = j
        } else if syn_is_digit(ch) || (ch == "-" && i + 1.0 < n && syn_is_digit(str_slice(ln, i + 1.0, 1.0))) {
            bind j = i + 1.0
            while j < n && (syn_is_digit(str_slice(ln, j, 1.0)) || str_slice(ln, j, 1.0) == "." || str_slice(ln, j, 1.0) == "e" || str_slice(ln, j, 1.0) == "-") { bind j = j + 1.0 }
            bind out = list_push(out, i)  bind out = list_push(out, j - i)  bind out = list_push(out, E_NUMBER)
            bind i = j
        } else if syn_is_word(ch) {
            bind j = i + 1.0
            while j < n && syn_is_word(str_slice(ln, j, 1.0)) { bind j = j + 1.0 }
            bind w = str_slice(ln, i, j - i)
            bind col = E_TEXT
            if w == "true" || w == "false" || w == "null" { bind col = E_CONST }
            bind out = list_push(out, i)  bind out = list_push(out, j - i)  bind out = list_push(out, col)
            bind i = j
        } else if ch == " " || ch == "\t" {
            bind i = i + 1.0
        } else {
            bind out = list_push(out, i)  bind out = list_push(out, 1.0)  bind out = list_push(out, E_OPER)
            bind i = i + 1.0
        }
    }
    return out
}

# tokenize `ln` for language `lang`
fn syn_line(lang, ln) {
    if lang == "ling" { return syn_ling(ln) }
    if lang == "toml" { return syn_toml(ln) }
    if lang == "md" { return syn_md(ln) }
    if lang == "rec" { return syn_rec(ln) }
    if lang == "json" { return syn_json(ln) }
    if lang == "cfg" { return syn_cfg(ln) }
    if lang == "sh" { return syn_sh(ln) }
    bind out = []
    if str_len(ln) > 0.5 {
        bind out = list_push(out, 0.0)  bind out = list_push(out, str_len(ln))  bind out = list_push(out, E_TEXT)
    }
    return out
}

# ── a light lint for .ling files (the Problems list) ─────────────────────────
# masks strings and comments, then: bracket balance, `let `, tabs in indentation.
# Results append "line|column|message" to the st list `problems`.
fn lint_ling(bi) {
    st_lclear("problems")
    bind n = buf_nlines(bi)
    bind depth = 0.0
    bind depth_par = 0.0
    bind depth_brk = 0.0
    bind i = 0.0
    while i < n {
        bind ln = buf_line(bi, i)
        bind m = str_len(ln)
        bind j = 0.0
        bind instr = false
        while j < m {
            bind ch = str_slice(ln, j, 1.0)
            if instr {
                if ch == "\\" { bind j = j + 1.0 }
                else if ch == "\"" { bind instr = false }
            } else {
                if ch == "\"" { bind instr = true }
                else if ch == "#" { bind j = m }
                else if ch == "/" && j + 1.0 < m && str_slice(ln, j + 1.0, 1.0) == "/" { bind j = m }
                else if ch == "{" { bind depth = depth + 1.0 }
                else if ch == "}" {
                    bind depth = depth - 1.0
                    if depth < 0.0 {
                        st_lpush("problems", num_str(i + 1.0) + "|" + num_str(j + 1.0) + "|unmatched '}'")
                        bind depth = 0.0
                    }
                }
                else if ch == "(" { bind depth_par = depth_par + 1.0 }
                else if ch == ")" { bind depth_par = max(0.0 - 1000.0, depth_par - 1.0) }
                else if ch == "[" { bind depth_brk = depth_brk + 1.0 }
                else if ch == "]" { bind depth_brk = depth_brk - 1.0 }
            }
            bind j = j + 1.0
        }
        if instr { st_lpush("problems", num_str(i + 1.0) + "|1|string does not close on this line") }
        bind t = str_trim(ln)
        if str_starts_with(t, "let ") { st_lpush("problems", num_str(i + 1.0) + "|1|`let` is not Ling - use `bind`") }
        if str_starts_with(ln, "\t") { st_lpush("problems", num_str(i + 1.0) + "|1|tab indentation - Ling code uses spaces") }
        bind i = i + 1.0
    }
    if depth > 0.5 { st_lpush("problems", num_str(n) + "|1|" + num_str(depth) + " unclosed '{'") }
    if depth_par > 0.5 { st_lpush("problems", num_str(n) + "|1|" + num_str(depth_par) + " unclosed '('") }
    if depth_brk > 0.5 { st_lpush("problems", num_str(n) + "|1|" + num_str(depth_brk) + " unclosed '['") }
    st_set("problems_for", buf_path(bi))
    return st_llen("problems")
}