Subject: Re: ls(1) support for multibyte chars
To: None <tech-userlevel@netbsd.org>
From: Christos Zoulas <christos@astron.com>
List: tech-userlevel
Date: 05/29/2007 16:43:01
In article <20070529163822.GE13298@danbala.tuwien.ac.at>,
Thomas Klausner  <wiz@NetBSD.org> wrote:
>-=-=-=-=-=-
>
>Hi!
>
>In a private discussion, SODA Noriyuki provided the attached patch for
>adding multibyte support to ls(1).
>
>Is it ok to commit?
> Thomas
>
>-=-=-=-=-=-
>
>Index: util.c
>===================================================================
>RCS file: /cvsroot/src/bin/ls/util.c,v
>retrieving revision 1.30
>diff -u -r1.30 util.c
>--- util.c	14 Dec 2006 14:15:26 -0000	1.30
>+++ util.c	31 Dec 2006 03:51:07 -0000
>@@ -44,7 +44,6 @@
> #include <sys/types.h>
> #include <sys/stat.h>
> 
>-#include <ctype.h>
> #include <err.h>
> #include <fts.h>
> #include <limits.h>
>@@ -52,6 +51,8 @@
> #include <stdlib.h>
> #include <string.h>
> #include <vis.h>
>+#include <wchar.h>
>+#include <wctype.h>
> 
> #include "ls.h"
> #include "extern.h"
>@@ -84,17 +85,76 @@
> 		/* NOTREACHED */
> }
> 
>+/*
>+ * The reasons why we don't use putwchar(wc) here are:
>+ * - If wc == L'\0', we need to restore the initial shift state, but
>+ *   the C language standard doesn't say that putwchar(L'\0') does.
>+ * - It isn't portable to mix a wide-oriented function (i.e. getwchar)
>+ *   with byte-oriented functions (printf et al.) in same FILE.
>+ */
>+static int
>+printwc(wchar_t wc, mbstate_t *pst)
>+{
>+	size_t size;
>+	char buf[MB_LEN_MAX];
>+
>+	size = wcrtomb(buf, wc, pst);
>+	if (size == (size_t)-1) /* This shouldn't happen, but for sure */
>+		return 0;
>+	if (wc == L'\0') {
>+		/* The following condition must be always true, but for sure */
>+		if (size > 0 && buf[size - 1] == '\0')
>+			--size;
>+	}
>+	if (size > 0)
>+		fwrite(buf, 1, size, stdout);
>+	return wc == L'\0' ? 0 : wcwidth(wc);
>+}
>+
> int
> printescaped(const char *src)
> {
>-	unsigned char c;
>-	int n;
>-
>-	for (n = 0; (c = *src) != '\0'; ++src, ++n)
>-		if (isprint(c))
>-			(void)putchar(c);
>-		else
>-			(void)putchar('?');
>+	int n = 0;
>+	mbstate_t src_state, stdout_state;
>+	/* The following +1 is to pass '\0' at the end of src to mbrtowc(). */
>+	const char *endptr = src + strlen(src) + 1;
>+
>+	/*
>+	 * We have to reset src_state each time in this function, because
>+	 * the codeset of src pathname may not match with current locale.
>+	 * Note that if we pass NULL instead of src_state to mbrtowc(),
>+	 * there is no way to reset the state.
>+	 */
>+	memset(&src_state, 0, sizeof(src_state));
>+	memset(&stdout_state, 0, sizeof(stdout_state));
>+	while (src < endptr) {
>+		wchar_t wc;
>+		size_t rv, span = endptr - src;
>+
>+		if (span > MB_CUR_MAX)
>+			span = MB_CUR_MAX;
>+		rv = mbrtowc(&wc, src, span, &src_state);
>+		if (rv == 0) { /* assert(wc == L'\0'); */
>+			/* The following may output a shift sequence. */
>+			n += printwc(wc, &stdout_state);
>+			break;
>+		}
>+		if (rv == (size_t)-1) { /* probably errno == EILSEQ */
>+			n += printwc(L'?', &stdout_state);
>+			/* try to skip 1byte, because there is no better way */
>+			src++;
>+			memset(&src_state, 0, sizeof(src_state));
>+		} else if (rv == (size_t)-2) {
>+			if (span < MB_CUR_MAX) { /* incomplete char */
>+				n += printwc(L'?', &stdout_state);
>+				break;
>+			}
>+			src += span; /* a redundant shift sequence? */
>+		} else {
>+			n += printwc(iswprint(wc) ? wc : L'?', &stdout_state);
>+			src += rv;
>+		}
>+	}
> 	return n;
> }
> 

Go for it.

christos