Changeset 7dcb7981 in mainline

Timestamp:

2011-08-19T11:35:08Z (13 years ago)

Author:

Martin Sucha <sucha14@…>

Branches:

lfn, master, serial, ticket/834-toolchain-update, topic/msim-upgrade, topic/simplify-dev-export

Children:

89660f2

Parents:

42a619b

Message:

Change bdsh tokenizer

Location:

uspace/app/bdsh

Files:

• tok.c (modified) (4 diffs)
• tok.h (modified) (2 diffs)

uspace/app/bdsh/tok.c

@@ -50 +50 @@
  * @param max_tokens number of elements of the out_tokens array
  */
-int tok_init(tokenizer_t *tok, char *input, char **out_tokens,
+int tok_init(tokenizer_t *tok, char *input, token_t *out_tokens,
     size_t max_tokens)
 {       
         tok->in = input;
         tok->in_offset = 0;
+        tok->last_in_offset = 0;
+        tok->in_char_offset = 0;
+        tok->last_in_char_offset = 0;
         
         tok->outtok = out_tokens;
@@ -201 +204 @@
 wchar_t tok_get_char(tokenizer_t *tok)
 {
+        tok->in_char_offset++;
         return str_decode(tok->in, &tok->in_offset, STR_NO_LIMIT);
 }
@@ -207 +211 @@
 wchar_t tok_look_char(tokenizer_t *tok)
 {
-        size_t old_offset = tok->in_offset;
+        off_t old_offset = tok->in_offset;
+        off_t old_char_offset = tok->in_char_offset;
         wchar_t ret = tok_get_char(tok);
         tok->in_offset = old_offset;
+        tok->in_char_offset = old_char_offset;
         return ret;
 }
@@ -231 +237 @@
         
         tok->outbuf[tok->outbuf_offset++] = 0;
-        tok->outtok[tok->outtok_offset++] = tok->outbuf + tok->outbuf_last_start;
+        token_t *tokinfo = &tok->outtok[tok->outtok_offset++]
+        tokinfo.text = tok->outbuf + tok->outbuf_last_start;
+        tokinfo.byte_start = tok->last_in_offset;
+        tokinfo.byte_length = tok->in_offset - tok->last_in_offset - 1;
+        tokinfo.char_start = tok->last_in_char_offset;
+        tokinfo.char_length = tok->in_char_offset - tok->last_in_char_offset
+            - 1;
+        tok->outtok[tok->outtok_offset]
         tok->outbuf_last_start = tok->outbuf_offset;
+        
+        /* We have consumed the first char of the next token already */
+        tok->last_in_offset = tok->in_offset-1;
+        tok->last_in_char_offset = tok->in_char_offset-1;
         
         return EOK;

uspace/app/bdsh/tok.h

@@ -30 +30 @@
 #define TOK_H
 
+typedef enum {
+        TOKTYPE_TEXT,
+        TOKTYPE_PIPE,
+        TOKTYPE_SPACE
+} token_type_t;
+
+typedef struct {
+        char *text;
+        off_t byte_start;
+        off_t char_start;
+        size_t byte_length;
+        size_t char_length;
+        token_type_t type;
+} token_t;
+
 typedef struct {
         char *in;
-        size_t in_offset;
+        off_t in_offset;
+        off_t last_in_offset;
+        off_t in_char_offset;
+        off_t last_in_char_offset;
         
         char *outbuf;
@@ -39 +57 @@
         size_t outbuf_last_start;
         
-        char **outtok;
+        token_t *outtok;
         size_t outtok_offset;
         size_t outtok_size;
 } tokenizer_t;
 
-extern int tok_init(tokenizer_t *, char *, char **, size_t);
+extern int tok_init(tokenizer_t *, char *, token_t *, size_t);
 extern void tok_fini(tokenizer_t *);
 extern int tok_tokenize(tokenizer_t *);