Method code for $compiler.tokenize()

[Turn off line numbering]

---

  1: arg text;
  2: var word, out, escaped, token, i, pre_count, open_form, str;
  3: 
  4: // break text into a list of tokens.
  5: if (type(text) == 'string)
  6:     text = [text];
  7: out = [];
  8: word = "";
  9: escaped = 0;
 10: 
 11: // pre_count is 0, except inside pre, when it counts the {}'s
 12: pre_count = 0;
 13: open_form = 0;
 14: for str in (text) {
 15:     str = str.explode(" ", 1);
 16:     for token in (str) {
 17:         refresh();
 18:         if (!token && !pre_count)
 19:             continue;
 20:         while ((i = token.match_regexp("[][{}=\:]"))) {
 21:             refresh();
 22:             i = i[1][1];
 23:             if (escaped) {
 24:                 escaped = 0;
 25:                 word = word + "\" + token.subrange(1, i);
 26:             } else if (pre_count) {
 27:                 if (token[i] == "{") {
 28:                     pre_count++;
 29:                     word += token.subrange(1, i);
 30:                 } else if (token[i] == "}") {
 31:                     pre_count--;
 32:                     if (pre_count) {
 33:                         word += token.subrange(1, i);
 34:                     } else {
 35:                         word += token.subrange(1, i - 1);
 36:                         out = word ? out + [word, token[i]] : out + [token[i]];
 37:                         word = "";
 38:                     }
 39:                 } else {
 40:                     word += token.subrange(1, i);
 41:                 }
 42:             } else {
 43:                 word += token.subrange(1, i - 1);
 44:                 open_form = token[i] == "{";
 45:                 if (token[i] == "\") {
 46:                     escaped = 1;
 47:                 } else {
 48:                     out = word ? out + [word, token[i]] : out + [token[i]];
 49:                     word = "";
 50:                 }
 51:             }
 52:             token = token.subrange(i + 1);
 53:         }
 54:         if (open_form && token == "quote") {
 55:             pre_count = 1;
 56:             open_form = 0;
 57:             out += [token];
 58:             token = "";
 59:             continue;
 60:         }
 61:         word += token;
 62:         if (escaped || pre_count) {
 63:             escaped = 0;
 64:             word += " ";
 65:         } else {
 66:             out = word ? out + [word, " "] : out + [" "];
 67:             word = "";
 68:         }
 69:     }
 70:     if (pre_count)
 71:         word = word ? word.subrange(1, word.length() - 1) + "
" : "
";
 72: }
 73: if (word)
 74:     out += [word];
 75: if (out) {
 76:     if (out.last() == " ")
 77:         out = out.subrange(1, out.length() - 1);
 78:     else
 79:         out = out.replace(out.length(), out.last().subrange(out.last().length() - 1));
 80: }
 81: return out;

---

// Created 17-Jul-1996 as a part of ColdCore, see: @help Credit