Changeset 0662451 in mainline for uspace/app/bdsh/tok.c
- Timestamp:
- 2011-08-19T14:44:49Z (13 years ago)
- Branches:
- lfn, master, serial, ticket/834-toolchain-update, topic/msim-upgrade, topic/simplify-dev-export
- Children:
- 5992e0e
- Parents:
- 89660f2
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
uspace/app/bdsh/tok.c
r89660f2 r0662451 42 42 static bool tok_pending_chars(tokenizer_t *); 43 43 static int tok_finish_string(tokenizer_t *); 44 static void tok_start_token(tokenizer_t *, token_type_t); 44 45 45 46 /** Initialize the token parser … … 61 62 tok->outtok = out_tokens; 62 63 tok->outtok_offset = 0; 63 /* Leave one slot for a null terminator */ 64 assert(max_tokens > 0); 65 tok->outtok_size = max_tokens - 1; 64 tok->outtok_size = max_tokens; 66 65 67 66 /* Prepare a buffer where all the token strings will be stored */ … … 90 89 91 90 /** Tokenize the input string into the tokens */ 92 int tok_tokenize(tokenizer_t *tok )91 int tok_tokenize(tokenizer_t *tok, size_t *tokens_length) 93 92 { 94 93 int rc; … … 98 97 while ((cur_char = tok_get_char(tok)) != 0) { 99 98 if (cur_char == ' ') { 100 /* Spaces delimit tokens, but are not processed in any way 101 * Push the token if there is any. 99 /* Push the token if there is any. 102 100 * There may not be any pending char for a token in case 103 101 * there are several spaces in the input. … … 109 107 } 110 108 } 109 tok_start_token(tok, TOKTYPE_SPACE); 110 /* Eat all spaces */ 111 while (tok_look_char(tok) == ' ') { 112 tok_push_char(tok, tok_get_char(tok)); 113 } 114 tok_push_token(tok); 115 111 116 } 112 117 else if (cur_char == '|') { … … 121 126 } 122 127 128 tok_start_token(tok, TOKTYPE_PIPE); 129 123 130 rc = tok_push_char(tok, '|'); 124 131 if (rc != EOK) { … … 135 142 * A literal quote is written as '' 136 143 */ 144 tok_start_token(tok, TOKTYPE_TEXT); 137 145 rc = tok_finish_string(tok); 138 146 if (rc != EOK) { … … 141 149 } 142 150 else { 151 if (!tok_pending_chars(tok)) { 152 tok_start_token(tok, TOKTYPE_TEXT); 153 } 143 154 /* If we are handling any other character, just append it to 144 155 * the current token. … … 159 170 } 160 171 161 /* We always have a space for the terminator, as we 162 * reserved it in tok_init */ 163 tok->outtok[tok->outtok_offset] = 0; 172 *tokens_length = tok->outtok_offset; 164 173 165 174 return EOK; … … 211 220 wchar_t tok_look_char(tokenizer_t *tok) 212 221 { 213 off_t old_offset = tok->in_offset;214 off_t old_char_offset = tok->in_char_offset;222 unsigned int old_offset = tok->in_offset; 223 unsigned int old_char_offset = tok->in_char_offset; 215 224 wchar_t ret = tok_get_char(tok); 216 225 tok->in_offset = old_offset; … … 225 234 } 226 235 236 void tok_start_token(tokenizer_t *tok, token_type_t type) 237 { 238 tok->current_type = type; 239 } 240 227 241 /** Push the current token to the output array */ 228 242 int tok_push_token(tokenizer_t *tok) … … 237 251 238 252 tok->outbuf[tok->outbuf_offset++] = 0; 239 token_t *tokinfo = &tok->outtok[tok->outtok_offset++] 240 tokinfo.text = tok->outbuf + tok->outbuf_last_start; 241 tokinfo.byte_start = tok->last_in_offset; 242 tokinfo.byte_length = tok->in_offset - tok->last_in_offset - 1; 243 tokinfo.char_start = tok->last_in_char_offset; 244 tokinfo.char_length = tok->in_char_offset - tok->last_in_char_offset 253 token_t *tokinfo = &tok->outtok[tok->outtok_offset++]; 254 tokinfo->type = tok->current_type; 255 tokinfo->text = tok->outbuf + tok->outbuf_last_start; 256 tokinfo->byte_start = tok->last_in_offset; 257 tokinfo->byte_length = tok->in_offset - tok->last_in_offset - 1; 258 tokinfo->char_start = tok->last_in_char_offset; 259 tokinfo->char_length = tok->in_char_offset - tok->last_in_char_offset 245 260 - 1; 246 tok->outtok[tok->outtok_offset]247 261 tok->outbuf_last_start = tok->outbuf_offset; 248 262
Note:
See TracChangeset
for help on using the changeset viewer.