Changeset 74c8da2c in mainline

Timestamp:

2009-03-24T14:41:31Z (15 years ago)

Author:

Martin Decky <martin@…>

Branches:

lfn, master, serial, ticket/834-toolchain-update, topic/msim-upgrade, topic/simplify-dev-export

Children:

eec616b

Parents:

4ccdcf6

Message:

more Unicode functions

Location:

kernel/generic

Files:

• include/string.h (modified) (1 diff)
• src/lib/string.c (modified) (8 diffs)

kernel/generic/include/string.h

@@ -38 +38 @@
 #include <typedefs.h>
 
-extern wchar_t utf8_decode(const char *str, index_t *index);
+#define UTF8_NO_LIMIT  ((index_t) -1)
+
+extern char invalch;
+
+extern wchar_t utf8_decode(const char *str, index_t *index, index_t limit);
+extern bool utf8_encode(const wchar_t ch, char *str, index_t *index, index_t limit);
+extern size_t utf8_count_bytes(const char *str, count_t count);
+extern bool ascii_check(const wchar_t ch);
+extern bool unicode_check(const wchar_t ch);
+
 extern size_t strlen(const char *str);
+extern size_t strlen_utf8(const char *str);
+extern size_t strlen_utf32(const wchar_t *str);
+
 extern int strcmp(const char *src, const char *dst);
 extern int strncmp(const char *src, const char *dst, size_t len);

kernel/generic/src/lib/string.c

@@ -43 +43 @@
 #include <console/kconsole.h>
 
+char invalch = '?';
+
 /** Decode a single UTF-8 character from a NULL-terminated string.
  *
@@ -52 +54 @@
  * @param index Index (counted in plain characters) where to start
  *              the decoding.
+ * @param limit Maximal allowed value of index.
  *
  * @return Decoded character in UTF-32 or '?' if the encoding is wrong.
  *
  */
-wchar_t utf8_decode(const char *str, index_t *index)
+wchar_t utf8_decode(const char *str, index_t *index, index_t limit)
 {
         uint8_t c1;           /* First plain character from str */
@@ -63 +66 @@
         uint8_t c4;           /* Fourth plain character from str */
         
+        if (*index > limit)
+                return invalch;
+        
         c1 = (uint8_t) str[*index];
         
@@ -68 +74 @@
                 /* Plain ASCII (code points 0 .. 127) */
                 return (wchar_t) c1;
-        } else if ((c1 & 0xe0) == 0xc0) {
+        }
+        
+        if ((c1 & 0xe0) == 0xc0) {
                 /* Code points 128 .. 2047 */
+                if (*index + 1 > limit)
+                        return invalch;
+                
                 c2 = (uint8_t) str[*index + 1];
                 if ((c2 & 0xc0) == 0x80) {
@@ -75 +86 @@
                         return ((wchar_t) ((c1 & 0x1f) << 6) | (c2 & 0x3f));
                 } else
-                        return ((wchar_t) '?');
-        } else if ((c1 & 0xf0) == 0xe0) {
+                        return invalch;
+        }
+        
+        if ((c1 & 0xf0) == 0xe0) {
                 /* Code points 2048 .. 65535 */
+                if (*index + 2 > limit)
+                        return invalch;
+                
                 c2 = (uint8_t) str[*index + 1];
                 if ((c2 & 0xc0) == 0x80) {
@@ -86 +102 @@
                                 return ((wchar_t) ((c1 & 0x0f) << 12) | ((c2 & 0x3f) << 6) | (c3 & 0x3f));
                         } else
-                                return ((wchar_t) '?');
+                                return invalch;
                 } else
-                        return ((wchar_t) '?');
-        } else if ((c1 & 0xf8) == 0xf0) {
+                        return invalch;
+        }
+        
+        if ((c1 & 0xf8) == 0xf0) {
                 /* Code points 65536 .. 1114111 */
+                if (*index + 3 > limit)
+                        return invalch;
+                
                 c2 = (uint8_t) str[*index + 1];
                 if ((c2 & 0xc0) == 0x80) {
@@ -102 +123 @@
                                         return ((wchar_t) ((c1 & 0x07) << 18) | ((c2 & 0x3f) << 12) | ((c3 & 0x3f) << 6) | (c4 & 0x3f));
                                 } else
-                                        return ((wchar_t) '?');
+                                        return invalch;
                         } else
-                                return ((wchar_t) '?');
+                                return invalch;
                 } else
-                        return ((wchar_t) '?');
-        }
-        
-        return ((wchar_t) '?');
-}
-
-/** Return number of characters in a string.
- *
- * @param str NULL terminated string.
+                        return invalch;
+        }
+        
+        return invalch;
+}
+
+/** Encode a single UTF-32 character as UTF-8
+ *
+ * Encode a single UTF-32 character as UTF-8 and store it into
+ * the given buffer at @index. Encoding starts at @index and
+ * this index is incremented if the UTF-8 character takes
+ * more than a single byte.
+ *
+ * @param ch    Input UTF-32 character.
+ * @param str   Output buffer.
+ * @param index Index (counted in plain characters) where to start
+ *              the encoding
+ * @param limit Maximal allowed value of index.
+ *
+ * @return True if the character was encoded or false if there is not
+ *         enought space in the output buffer or the character is invalid
+ *         Unicode code point.
+ *
+ */
+bool utf8_encode(const wchar_t ch, char *str, index_t *index, index_t limit)
+{
+        if (*index > limit)
+                return false;
+        
+        if ((ch >= 0) && (ch <= 127)) {
+                /* Plain ASCII (code points 0 .. 127) */
+                str[*index] = ch & 0x7f;
+                return true;
+        }
+        
+        if ((ch >= 128) && (ch <= 2047)) {
+                /* Code points 128 .. 2047 */
+                if (*index + 1 > limit)
+                        return false;
+                
+                str[*index] = 0xc0 | ((ch >> 6) & 0x1f);
+                (*index)++;
+                str[*index] = 0x80 | (ch & 0x3f);
+                return true;
+        }
+        
+        if ((ch >= 2048) && (ch <= 65535)) {
+                /* Code points 2048 .. 65535 */
+                if (*index + 2 > limit)
+                        return false;
+                
+                str[*index] = 0xe0 | ((ch >> 12) & 0x0f);
+                (*index)++;
+                str[*index] = 0x80 | ((ch >> 6) & 0x3f);
+                (*index)++;
+                str[*index] = 0x80 | (ch & 0x3f);
+                return true;
+        }
+        
+        if ((ch >= 65536) && (ch <= 1114111)) {
+                /* Code points 65536 .. 1114111 */
+                if (*index + 3 > limit)
+                        return false;
+                
+                str[*index] = 0xf0 | ((ch >> 18) & 0x07);
+                (*index)++;
+                str[*index] = 0x80 | ((ch >> 12) & 0x3f);
+                (*index)++;
+                str[*index] = 0x80 | ((ch >> 6) & 0x3f);
+                (*index)++;
+                str[*index] = 0x80 | (ch & 0x3f);
+                return true;
+        }
+        
+        return false;
+}
+
+/** Get bytes used by UTF-8 characters.
+ *
+ * Get the number of bytes (count of plain characters) which
+ * are used by a given count of UTF-8 characters in a string.
+ * As UTF-8 encoding is multibyte, there is no constant
+ * correspondence between number of characters and used bytes.
+ *
+ * @param str   UTF-8 string to consider.
+ * @param count Number of UTF-8 characters to count.
+ *
+ * @return Number of bytes used by the characters.
+ *
+ */
+size_t utf8_count_bytes(const char *str, count_t count)
+{
+        size_t size = 0;
+        index_t index = 0;
+        
+        while ((utf8_decode(str, &index, UTF8_NO_LIMIT) != 0) && (size < count)) {
+                size++;
+                index++;
+        }
+        
+        return index;
+}
+
+/** Check whether character is plain ASCII.
+ *
+ * @return True if character is plain ASCII.
+ *
+ */
+bool ascii_check(const wchar_t ch)
+{
+        if ((ch >= 0) && (ch <= 127))
+                return true;
+        
+        return false;
+}
+
+/** Check whether character is Unicode.
+ *
+ * @return True if character is valid Unicode code point.
+ *
+ */
+bool unicode_check(const wchar_t ch)
+{
+        if ((ch >= 0) && (ch <= 1114111))
+                return true;
+        
+        return false;
+}
+
+/** Return number of plain characters in a string.
+ *
+ * @param str NULL-terminated string.
  *
  * @return Number of characters in str.
@@ -121 +265 @@
 size_t strlen(const char *str)
 {
-        int i;
-        
-        for (i = 0; str[i]; i++);
-        
-        return i;
+        size_t size;
+        for (size = 0; str[size]; size++);
+        
+        return size;
+}
+
+/** Return number of UTF-8 characters in a string.
+ *
+ * @param str NULL-terminated UTF-8 string.
+ *
+ * @return Number of UTF-8 characters in str.
+ *
+ */
+size_t strlen_utf8(const char *str)
+{
+        size_t size = 0;
+        index_t index = 0;
+        
+        while (utf8_decode(str, &index, UTF8_NO_LIMIT) != 0) {
+                size++;
+                index++;
+        }
+        
+        return size;
+}
+
+/** Return number of UTF-32 characters in a string.
+ *
+ * @param str NULL-terminated UTF-32 string.
+ *
+ * @return Number of UTF-32 characters in str.
+ *
+ */
+size_t strlen_utf32(const wchar_t *str)
+{
+        size_t size;
+        for (size = 0; str[size]; size++);
+        
+        return size;
 }