Method code for $compiler.tokenize()

[Turn on line numbering]

---

arg text;
var word, out, escaped, token, i, pre_count, open_form, str;

// break text into a list of tokens.
if (type(text) == 'string)
    text = [text];
out = [];
word = "";
escaped = 0;

// pre_count is 0, except inside pre, when it counts the {}'s
pre_count = 0;
open_form = 0;
for str in (text) {
    str = str.explode(" ", 1);
    for token in (str) {
        refresh();
        if (!token && !pre_count)
            continue;
        while ((i = token.match_regexp("[][{}=\:]"))) {
            refresh();
            i = i[1][1];
            if (escaped) {
                escaped = 0;
                word = word + "\" + token.subrange(1, i);
            } else if (pre_count) {
                if (token[i] == "{") {
                    pre_count++;
                    word += token.subrange(1, i);
                } else if (token[i] == "}") {
                    pre_count--;
                    if (pre_count) {
                        word += token.subrange(1, i);
                    } else {
                        word += token.subrange(1, i - 1);
                        out = word ? out + [word, token[i]] : out + [token[i]];
                        word = "";
                    }
                } else {
                    word += token.subrange(1, i);
                }
            } else {
                word += token.subrange(1, i - 1);
                open_form = token[i] == "{";
                if (token[i] == "\") {
                    escaped = 1;
                } else {
                    out = word ? out + [word, token[i]] : out + [token[i]];
                    word = "";
                }
            }
            token = token.subrange(i + 1);
        }
        if (open_form && token == "quote") {
            pre_count = 1;
            open_form = 0;
            out += [token];
            token = "";
            continue;
        }
        word += token;
        if (escaped || pre_count) {
            escaped = 0;
            word += " ";
        } else {
            out = word ? out + [word, " "] : out + [" "];
            word = "";
        }
    }
    if (pre_count)
        word = word ? word.subrange(1, word.length() - 1) + "
" : "
";
}
if (word)
    out += [word];
if (out) {
    if (out.last() == " ")
        out = out.subrange(1, out.length() - 1);
    else
        out = out.replace(out.length(), out.last().subrange(out.last().length() - 1));
}
return out;

---

// Created 17-Jul-1996 as a part of ColdCore, see: @help Credit