libbb: better unicode width support. Hopefully fixes bug 839.
Also opens up a possibility to make other unicode stuff smaller and more correct later. but: function old new delta static.combining - 516 +516 bb_wcwidth - 328 +328 unicode_cut_nchars - 141 +141 mbstowc_internal - 93 +93 in_table - 78 +78 cal_main 899 961 +62 static.combining0x10000 - 40 +40 unicode_strlen - 31 +31 bb_mbstrlen 31 - -31 bb_mbstowcs 173 102 -71 ------------------------------------------------------------------------------ (add/remove: 7/1 grow/shrink: 1/1 up/down: 1289/-102) Total: 1187 bytes Uses code of Markus Kuhn, which is in public domain: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c "Permission to use, copy, modify, and distribute this software for any purpose and without fee is hereby granted. The author disclaims all warranties with regard to this software." Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com>
This commit is contained in:
Denys Vlasenko
parent
5da9f96ad8
commit
9f93d62192
10 changed files with 410 additions and 98 deletions
210
libbb/unicode.c
210
libbb/unicode.c
|
|
@ -9,22 +9,19 @@
|
|||
#include "libbb.h"
|
||||
#include "unicode.h"
|
||||
|
||||
/* If it's not a constant... */
|
||||
/* If it's not #defined as a constant in unicode.h... */
|
||||
#ifndef unicode_status
|
||||
uint8_t unicode_status;
|
||||
#endif
|
||||
|
||||
size_t FAST_FUNC bb_mbstrlen(const char *string)
|
||||
{
|
||||
size_t width = mbstowcs(NULL, string, INT_MAX);
|
||||
if (width == (size_t)-1L)
|
||||
return strlen(string);
|
||||
return width;
|
||||
}
|
||||
/* This file is compiled only if FEATURE_ASSUME_UNICODE is on.
|
||||
* We check other options and decide whether to use libc support
|
||||
* via locale, or use our own logic:
|
||||
*/
|
||||
|
||||
#if ENABLE_LOCALE_SUPPORT
|
||||
|
||||
/* Unicode support using libc */
|
||||
/* Unicode support using libc locale support. */
|
||||
|
||||
void FAST_FUNC init_unicode(void)
|
||||
{
|
||||
|
|
@ -34,12 +31,12 @@ void FAST_FUNC init_unicode(void)
|
|||
if (unicode_status != UNICODE_UNKNOWN)
|
||||
return;
|
||||
|
||||
unicode_status = bb_mbstrlen(unicode_0x394) == 1 ? UNICODE_ON : UNICODE_OFF;
|
||||
unicode_status = unicode_strlen(unicode_0x394) == 1 ? UNICODE_ON : UNICODE_OFF;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
/* Crude "locale support" which knows only C and Unicode locales */
|
||||
/* Homegrown Unicode support. It knows only C and Unicode locales. */
|
||||
|
||||
# if ENABLE_FEATURE_CHECK_UNICODE_IN_ENV
|
||||
void FAST_FUNC init_unicode(void)
|
||||
|
|
@ -93,7 +90,6 @@ static size_t wcrtomb_internal(char *s, wchar_t wc)
|
|||
s[0] = wc | (uint8_t)(0x3f00 >> n);
|
||||
return n;
|
||||
}
|
||||
|
||||
size_t FAST_FUNC wcrtomb(char *s, wchar_t wc, mbstate_t *ps UNUSED_PARAM)
|
||||
{
|
||||
if (unicode_status != UNICODE_ON) {
|
||||
|
|
@ -103,7 +99,6 @@ size_t FAST_FUNC wcrtomb(char *s, wchar_t wc, mbstate_t *ps UNUSED_PARAM)
|
|||
|
||||
return wcrtomb_internal(s, wc);
|
||||
}
|
||||
|
||||
size_t FAST_FUNC wcstombs(char *dest, const wchar_t *src, size_t n)
|
||||
{
|
||||
size_t org_n = n;
|
||||
|
|
@ -144,6 +139,51 @@ size_t FAST_FUNC wcstombs(char *dest, const wchar_t *src, size_t n)
|
|||
return org_n - n;
|
||||
}
|
||||
|
||||
static const char *mbstowc_internal(wchar_t *res, const char *src)
|
||||
{
|
||||
int bytes;
|
||||
unsigned c = (unsigned char) *src++;
|
||||
|
||||
if (c <= 0x7f) {
|
||||
*res = c;
|
||||
return src;
|
||||
}
|
||||
|
||||
/* 80-7FF -> 110yyyxx 10xxxxxx */
|
||||
/* 800-FFFF -> 1110yyyy 10yyyyxx 10xxxxxx */
|
||||
/* 10000-1FFFFF -> 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx */
|
||||
/* 200000-3FFFFFF -> 111110tt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */
|
||||
/* 4000000-FFFFFFFF -> 111111tt 10tttttt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */
|
||||
bytes = 0;
|
||||
do {
|
||||
c <<= 1;
|
||||
bytes++;
|
||||
} while ((c & 0x80) && bytes < 6);
|
||||
if (bytes == 1)
|
||||
return NULL;
|
||||
c = (uint8_t)(c) >> bytes;
|
||||
|
||||
while (--bytes) {
|
||||
unsigned ch = (unsigned char) *src++;
|
||||
if ((ch & 0xc0) != 0x80) {
|
||||
return NULL;
|
||||
}
|
||||
c = (c << 6) + (ch & 0x3f);
|
||||
}
|
||||
|
||||
/* TODO */
|
||||
/* Need to check that c isn't produced by overlong encoding */
|
||||
/* Example: 11000000 10000000 converts to NUL */
|
||||
/* 11110000 10000000 10000100 10000000 converts to 0x100 */
|
||||
/* correct encoding: 11000100 10000000 */
|
||||
if (c <= 0x7f) { /* crude check */
|
||||
return NULL;
|
||||
//or maybe 0xfffd; /* replacement character */
|
||||
}
|
||||
|
||||
*res = c;
|
||||
return src;
|
||||
}
|
||||
size_t FAST_FUNC mbstowcs(wchar_t *dest, const char *src, size_t n)
|
||||
{
|
||||
size_t org_n = n;
|
||||
|
|
@ -162,58 +202,20 @@ size_t FAST_FUNC mbstowcs(wchar_t *dest, const char *src, size_t n)
|
|||
}
|
||||
|
||||
while (n) {
|
||||
int bytes;
|
||||
unsigned c = (unsigned char) *src++;
|
||||
|
||||
if (c <= 0x7f) {
|
||||
if (dest)
|
||||
*dest++ = c;
|
||||
if (c == '\0')
|
||||
break;
|
||||
n--;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* 80-7FF -> 110yyyxx 10xxxxxx */
|
||||
/* 800-FFFF -> 1110yyyy 10yyyyxx 10xxxxxx */
|
||||
/* 10000-1FFFFF -> 11110zzz 10zzyyyy 10yyyyxx 10xxxxxx */
|
||||
/* 200000-3FFFFFF -> 111110tt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */
|
||||
/* 4000000-FFFFFFFF -> 111111tt 10tttttt 10zzzzzz 10zzyyyy 10yyyyxx 10xxxxxx */
|
||||
bytes = 0;
|
||||
do {
|
||||
c <<= 1;
|
||||
bytes++;
|
||||
} while ((c & 0x80) && bytes < 6);
|
||||
if (bytes == 1)
|
||||
wchar_t wc;
|
||||
const char *rc = mbstowc_internal(dest ? dest : &wc, src);
|
||||
if (rc == NULL) /* error */
|
||||
return (size_t) -1L;
|
||||
c = (uint8_t)(c) >> bytes;
|
||||
|
||||
while (--bytes) {
|
||||
unsigned ch = (unsigned char) *src++;
|
||||
if ((ch & 0xc0) != 0x80) {
|
||||
return (size_t) -1L;
|
||||
}
|
||||
c = (c << 6) + (ch & 0x3f);
|
||||
}
|
||||
|
||||
/* TODO */
|
||||
/* Need to check that c isn't produced by overlong encoding */
|
||||
/* Example: 11000000 10000000 converts to NUL */
|
||||
/* 11110000 10000000 10000100 10000000 converts to 0x100 */
|
||||
/* correct encoding: 11000100 10000000 */
|
||||
if (c <= 0x7f) { /* crude check */
|
||||
return (size_t) -1L;
|
||||
//or maybe: c = 0xfffd; /* replacement character */
|
||||
}
|
||||
|
||||
if (dest)
|
||||
*dest++ = c;
|
||||
dest++;
|
||||
n--;
|
||||
}
|
||||
|
||||
return org_n - n;
|
||||
}
|
||||
|
||||
#include "unicode_wcwidth.c"
|
||||
|
||||
int FAST_FUNC iswspace(wint_t wc)
|
||||
{
|
||||
return (unsigned)wc <= 0x7f && isspace(wc);
|
||||
|
|
@ -229,4 +231,98 @@ int FAST_FUNC iswpunct(wint_t wc)
|
|||
return (unsigned)wc <= 0x7f && ispunct(wc);
|
||||
}
|
||||
|
||||
#endif /* Homegrown Unicode support */
|
||||
|
||||
|
||||
/* The rest is mostly same for libc and for "homegrown" support */
|
||||
|
||||
size_t FAST_FUNC unicode_strlen(const char *string)
|
||||
{
|
||||
size_t width = mbstowcs(NULL, string, INT_MAX);
|
||||
if (width == (size_t)-1L)
|
||||
return strlen(string);
|
||||
return width;
|
||||
}
|
||||
|
||||
char* FAST_FUNC unicode_cut_nchars(unsigned width, const char *src)
|
||||
{
|
||||
char *dst;
|
||||
unsigned dst_len;
|
||||
|
||||
if (unicode_status != UNICODE_ON)
|
||||
return xasprintf("%-*.*s", width, width, src);
|
||||
|
||||
dst = NULL;
|
||||
dst_len = 0;
|
||||
while (1) {
|
||||
int w;
|
||||
wchar_t wc;
|
||||
|
||||
dst = xrealloc(dst, dst_len + 2 * MB_CUR_MAX);
|
||||
#if ENABLE_LOCALE_SUPPORT
|
||||
{
|
||||
mbstate_t mbst = { 0 };
|
||||
ssize_t rc = mbsrtowcs(&wc, &src, 1, &mbst);
|
||||
if (rc <= 0) /* error, or end-of-string */
|
||||
break;
|
||||
}
|
||||
#else
|
||||
src = mbstowc_internal(&wc, src);
|
||||
if (!src || wc == 0) /* error, or end-of-string */
|
||||
break;
|
||||
#endif
|
||||
w = wcwidth(wc);
|
||||
if (w < 0) /* non-printable wchar */
|
||||
break;
|
||||
width -= w;
|
||||
if ((int)width < 0) { /* string is longer than width */
|
||||
width += w;
|
||||
while (width) {
|
||||
dst[dst_len++] = ' ';
|
||||
width--;
|
||||
}
|
||||
break;
|
||||
}
|
||||
#if ENABLE_LOCALE_SUPPORT
|
||||
{
|
||||
mbstate_t mbst = { 0 };
|
||||
dst_len += wcrtomb(&dst[dst_len], wc, &mbst);
|
||||
}
|
||||
#else
|
||||
dst_len += wcrtomb_internal(&dst[dst_len], wc);
|
||||
#endif
|
||||
}
|
||||
dst[dst_len] = '\0';
|
||||
return dst;
|
||||
}
|
||||
|
||||
unsigned FAST_FUNC unicode_padding_to_width(unsigned width, const char *src)
|
||||
{
|
||||
if (unicode_status != UNICODE_ON) {
|
||||
return width - strnlen(src, width);
|
||||
}
|
||||
|
||||
while (1) {
|
||||
int w;
|
||||
wchar_t wc;
|
||||
|
||||
#if ENABLE_LOCALE_SUPPORT
|
||||
{
|
||||
mbstate_t mbst = { 0 };
|
||||
ssize_t rc = mbsrtowcs(&wc, &src, 1, &mbst);
|
||||
if (rc <= 0) /* error, or end-of-string */
|
||||
return width;
|
||||
}
|
||||
#else
|
||||
src = mbstowc_internal(&wc, src);
|
||||
if (!src || wc == 0) /* error, or end-of-string */
|
||||
return width;
|
||||
#endif
|
||||
w = wcwidth(wc);
|
||||
if (w < 0) /* non-printable wchar */
|
||||
return width;
|
||||
width -= w;
|
||||
if ((int)width <= 0) /* string is longer than width */
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue