charsetConv.c - platform/external/mtools - Git at Google

 /*  Copyright 2008,2009 Alain Knaff.
  *  This file is part of mtools.
  *
  *  Mtools is free software: you can redistribute it and/or modify
  *  it under the terms of the GNU General Public License as published by
  *  the Free Software Foundation, either version 3 of the License, or
  *  (at your option) any later version.
  *
  *  Mtools is distributed in the hope that it will be useful,
  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  *  GNU General Public License for more details.
  *
  *  You should have received a copy of the GNU General Public License
  *  along with Mtools.  If not, see <http://www.gnu.org/licenses/>.
  *
  * Various character set conversions used by mtools
  */
 #include "sysincludes.h"
 #include "msdos.h"
 #include "mtools.h"

 #include <stdio.h>
 #include <errno.h>
 #include <stdlib.h>
 #include "file_name.h"


 #ifdef HAVE_ICONV_H
 #include <iconv.h>

 struct doscp_t {
 	iconv_t from;
 	iconv_t to;
 };

 static const char *wcharCp=NULL;

 static const char* wcharTries[] = {
 	"WCHAR_T",
 	"UTF-32BE", "UTF-32LE",
 	"UTF-16BE", "UTF-16LE",
 	"UTF-32", "UTF-16",
 	"UCS-4BE", "UCS-4LE",
 	"UCS-2BE", "UCS-2LE",
 	"UCS-4", "UCS-2"
 };

 static const char *asciiTries[] = {
 	"ASCII", "ASCII-GR", "ISO8859-1"
 };

 static const wchar_t *testString = L"ab";

 static int try(const char *testCp) {
 	size_t res;
 	char *inbuf = (char *)testString;
 	size_t inbufLen = 2*sizeof(wchar_t);
 	char outbuf[3];
 	char *outbufP = outbuf;
 	size_t outbufLen = 2*sizeof(char);
 	iconv_t test = 0;
 	size_t i;

 	for(i=0; i < sizeof(asciiTries) / sizeof(asciiTries[0]); i++) {
 		test = iconv_open(asciiTries[i], testCp);
 		if(test != (iconv_t) -1)
 			break;
 	}
 	if(test == (iconv_t) -1)
 		goto fail0;
 	res = iconv(test,
 		    &inbuf, &inbufLen,
 		    &outbufP, &outbufLen);
 	if(res != 0 || outbufLen != 0 || inbufLen != 0)
 		goto fail;
 	if(memcmp(outbuf, "ab", 2))
 		goto fail;
 	/* fprintf(stderr, "%s ok\n", testCp); */
 	return 1;
  fail:
 	iconv_close(test);
  fail0:
 	/*fprintf(stderr, "%s fail\n", testCp);*/
 	return 0;
 }

 static const char *getWcharCp(void) {
 	unsigned int i;
 	if(wcharCp != NULL)
 		return wcharCp;
 	for(i=0; i< sizeof(wcharTries) / sizeof(wcharTries[0]); i++) {
 		if(try(wcharTries[i]))
 			return (wcharCp=wcharTries[i]);
 	}
 	fprintf(stderr, "No codepage found for wchar_t\n");
 	return NULL;
 }


 doscp_t *cp_open(unsigned int codepage)
 {
 	char dosCp[17];
 	doscp_t *ret;
 	iconv_t from;
 	iconv_t to;

 	if(codepage == 0)
 		codepage = mtools_default_codepage;
 	if(codepage > 9999) {
 		fprintf(stderr, "Bad codepage %d\n", codepage);
 		return NULL;
 	}

 	if(getWcharCp() == NULL)
 		return NULL;

 	sprintf(dosCp, "CP%d", codepage);
 	from = iconv_open(wcharCp, dosCp);
 	if(from == (iconv_t)-1) {
 		fprintf(stderr, "Error converting to codepage %d %s\n",
 			codepage, strerror(errno));
 		return NULL;
 	}

 	sprintf(dosCp, "CP%d//TRANSLIT", codepage);
 	to   =  iconv_open(dosCp, wcharCp);
 	if(to == (iconv_t)-1) {
 		/* Transliteration not supported? */
 		sprintf(dosCp, "CP%d", codepage);
 		to   =  iconv_open(dosCp, wcharCp);
 	}
 	if(to == (iconv_t)-1) {
 		iconv_close(from);
 		fprintf(stderr, "Error converting to codepage %d %s\n",
 			codepage, strerror(errno));
 		return NULL;
 	}

 	ret = New(doscp_t);
 	if(ret == NULL)
 		return ret;
 	ret->from = from;
 	ret->to   = to;
 	return ret;
 }

 void cp_close(doscp_t *cp)
 {
 	iconv_close(cp->to);
 	iconv_close(cp->from);
 	free(cp);
 }

 size_t dos_to_wchar(doscp_t *cp, const char *dos, wchar_t *wchar, size_t len)
 {
 	size_t r;
 	size_t in_len=len;
 	size_t out_len=len*sizeof(wchar_t);
 	wchar_t *dptr=wchar;
 	char *dos2 = (char *) dos; /* Magic to be able to call iconv with its
 				      buggy prototype */
 	r=iconv(cp->from, &dos2, &in_len, (char **)&dptr, &out_len);
 	if(r == (size_t) -1)
 		return r;
 	*dptr = L'\0';
 	return (size_t) (dptr-wchar);
 }

 /**
  * Converts len wide character to destination. Caller's responsibility to
  * ensure that dest is large enough.
  * mangled will be set if there has been an untranslatable character.
  */
 static size_t safe_iconv(iconv_t conv, const wchar_t *wchar, char *dest,
 		      size_t in_len, size_t out_len, int *mangled)
 {
 	size_t r;
 	unsigned int i;
 	char *dptr = dest;
 	size_t len;

 	in_len=in_len*sizeof(wchar_t);

 	while(in_len > 0 && out_len > 0) {
 		r=iconv(conv, (char**)&wchar, &in_len, &dptr, &out_len);
 		if(r == (size_t) -1 || errno != EILSEQ) {
 			/* everything transformed, or error that is _not_ a bad
 			 * character */
 			break;
 		}
 		*mangled |= 1;

 		if(out_len <= 0)
 			break;
 		if(dptr)
 			*dptr++ = '_';
 		in_len -= sizeof(wchar_t);

 		wchar++;
 		out_len--;
 	}

 	len = (size_t) (dptr-dest); /* how many dest characters have there been
 				       generated */

 	/* eliminate question marks which might have been formed by
 	   untransliterable characters */
 	for(i=0; i<len; i++) {
 		if(dest[i] == '?') {
 			dest[i] = '_';
 			*mangled |= 1;
 		}
 	}
 	return len;
 }

 void wchar_to_dos(doscp_t *cp,
 		  wchar_t *wchar, char *dos, size_t len, int *mangled)
 {
 	safe_iconv(cp->to, wchar, dos, len, len, mangled);
 }

 #else

 #include "codepage.h"

 struct doscp_t {
 	unsigned char *from_dos;
 	unsigned char to_dos[0x80];
 };

 doscp_t *cp_open(unsigned int codepage)
 {
 	doscp_t *ret;
 	int i;
 	Codepage_t *cp;

 	if(codepage == 0)
 		codepage = 850;

 	ret = New(doscp_t);
 	if(ret == NULL)
 		return ret;

 	for(cp=codepages; cp->nr ; cp++)
 		if(cp->nr == codepage) {
 			ret->from_dos = cp->tounix;
 			break;
 		}

 	if(ret->from_dos == NULL) {
 		fprintf(stderr, "Bad codepage %d\n", codepage);
 		free(ret);
 		return NULL;
 	}

 	for(i=0; i<0x80; i++) {
 		char native = ret->from_dos[i];
 		if(! (native & 0x80))
 			continue;
 		ret->to_dos[native & 0x7f] = 0x80 | i;
 	}
 	return ret;
 }

 void cp_close(doscp_t *cp)
 {
 	free(cp);
 }

 size_t dos_to_wchar(doscp_t *cp, const char *dos, wchar_t *wchar, size_t len)
 {
 	int i;

 	for(i=0; i<len && dos[i]; i++) {
 		char c = dos[i];
 		if(c >= ' ' && c <= '~')
 			wchar[i] = c;
 		else {
 			wchar[i] = cp->from_dos[c & 0x7f];
 		}
 	}
 	wchar[i] = '\0';
 	return i;
 }


 void wchar_to_dos(doscp_t *cp,
 		  wchar_t *wchar, char *dos, size_t len, int *mangled)
 {
 	int i;
 	for(i=0; i<len && wchar[i]; i++) {
 		char c = wchar[i];
 		if(c >= ' ' && c <= '~')
 			dos[i] = c;
 		else {
 			dos[i] = cp->to_dos[c & 0x7f];
 			if(dos[i] == '\0') {
 				dos[i]='_';
 				*mangled=1;
 			}
 		}
 	}
 }

 #endif


 #ifndef HAVE_WCHAR_H

 typedef int mbstate_t;

 static inline size_t wcrtomb(char *s, wchar_t wc, mbstate_t *ps)
 {
 	*s = wc;
 	return 1;
 }

 static inline size_t mbrtowc(wchar_t *pwc, const char *s,
 			     size_t n, mbstate_t *ps)
 {
 	*pwc = *s;
 	return 1;
 }

 #endif

 #ifdef HAVE_ICONV_H

 #include <langinfo.h>

 static iconv_t to_native = NULL;

 static void initialize_to_native(void)
 {
 	char *li, *cp;
 	size_t len;
 	if(to_native != NULL)
 		return;
 	li = nl_langinfo(CODESET);
 	len = strlen(li) + 11;
 	if(getWcharCp() == NULL)
 		exit(1);
 	cp = safe_malloc(len);
 	strcpy(cp, li);
 	strcat(cp, "//TRANSLIT");
 	to_native = iconv_open(cp, wcharCp);
 	if(to_native == (iconv_t) -1)
 		to_native = iconv_open(li, wcharCp);
 	if(to_native == (iconv_t) -1)
 		fprintf(stderr, "Could not allocate iconv for %s\n", cp);
 	free(cp);
 	if(to_native == (iconv_t) -1)
 		exit(1);
 }


 #endif


 /**
  * Convert wchar string to native, converting at most len wchar characters
  * Returns number of generated native characters
  */
 size_t wchar_to_native(const wchar_t *wchar, char *native, size_t len,
 		       size_t out_len)
 {
 #ifdef HAVE_ICONV_H
 	int mangled;
 	size_t r;
 	initialize_to_native();
 	len = wcsnlen(wchar,len);
 	r=safe_iconv(to_native, wchar, native, len, out_len, &mangled);
 	native[r]='\0';
 	return r;
 #else
 	int i;
 	char *dptr = native;
 	mbstate_t ps;
 	memset(&ps, 0, sizeof(ps));
 	for(i=0; i<len && wchar[i] != 0; i++) {
 		size_t r = wcrtomb(dptr, wchar[i], &ps);
 		if(r == (size_t) -1 && errno == EILSEQ) {
 			r=1;
 			*dptr='_';
 		}
 		dptr+=r;
 	}
 	*dptr='\0';
 	return dptr-native;
 #endif
 }

 /**
  * Convert native string to wchar string, generating at most len wchar
  * characters. If end is supplied, stop conversion when source pointer
  * exceeds end. Returns number of generated wchars
  */
 size_t native_to_wchar(const char *native, wchar_t *wchar, size_t len,
 		       const char *end, int *mangled)
 {
 	mbstate_t ps;
 	unsigned int i;
 	memset(&ps, 0, sizeof(ps));

 	for(i=0; i<len && (native < end || !end); i++) {
 		size_t r = mbrtowc(wchar+i, native, len, &ps);
 		if(r == (size_t) -1) {
 			/* Unconvertible character. Just pretend it's Latin1
 			   encoded (if valid Latin1 character) or substitute
 			   with an underscore if not
 			*/
 			char c = *native;
 			if(c >= '\xa0' && c < '\xff')
 				wchar[i] = c & 0xff;
 			else
 				wchar[i] = '_';
 			memset(&ps, 0, sizeof(ps));
 			r=1;
 		}
 		if(r == 0)
 			break;
 		native += r;
 	}
 	if(mangled && ((end && native < end) || (!end && *native &&  i == len)))
 		*mangled |= 3;
 	wchar[i]='\0';
 	return i;
 }
	/* Copyright 2008,2009 Alain Knaff.
	* This file is part of mtools.
	*
	* Mtools is free software: you can redistribute it and/or modify
	* it under the terms of the GNU General Public License as published by
	* the Free Software Foundation, either version 3 of the License, or
	* (at your option) any later version.
	*
	* Mtools is distributed in the hope that it will be useful,
	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	* GNU General Public License for more details.
	*
	* You should have received a copy of the GNU General Public License
	* along with Mtools. If not, see <http://www.gnu.org/licenses/>.
	*
	* Various character set conversions used by mtools
	*/
	#include "sysincludes.h"
	#include "msdos.h"
	#include "mtools.h"

	#include <stdio.h>
	#include <errno.h>
	#include <stdlib.h>
	#include "file_name.h"


	#ifdef HAVE_ICONV_H
	#include <iconv.h>

	struct doscp_t {
	iconv_t from;
	iconv_t to;
	};

	static const char *wcharCp=NULL;

	static const char* wcharTries[] = {
	"WCHAR_T",
	"UTF-32BE", "UTF-32LE",
	"UTF-16BE", "UTF-16LE",
	"UTF-32", "UTF-16",
	"UCS-4BE", "UCS-4LE",
	"UCS-2BE", "UCS-2LE",
	"UCS-4", "UCS-2"
	};

	static const char *asciiTries[] = {
	"ASCII", "ASCII-GR", "ISO8859-1"
	};

	static const wchar_t *testString = L"ab";

	static int try(const char *testCp) {
	size_t res;
	char inbuf = (char )testString;
	size_t inbufLen = 2*sizeof(wchar_t);
	char outbuf[3];
	char *outbufP = outbuf;
	size_t outbufLen = 2*sizeof(char);
	iconv_t test = 0;
	size_t i;

	for(i=0; i < sizeof(asciiTries) / sizeof(asciiTries[0]); i++) {
	test = iconv_open(asciiTries[i], testCp);
	if(test != (iconv_t) -1)
	break;
	}
	if(test == (iconv_t) -1)
	goto fail0;
	res = iconv(test,
	&inbuf, &inbufLen,
	&outbufP, &outbufLen);
	if(res != 0 \|\| outbufLen != 0 \|\| inbufLen != 0)
	goto fail;
	if(memcmp(outbuf, "ab", 2))
	goto fail;
	/* fprintf(stderr, "%s ok\n", testCp); */
	return 1;
	fail:
	iconv_close(test);
	fail0:
	/fprintf(stderr, "%s fail\n", testCp);/
	return 0;
	}

	static const char *getWcharCp(void) {
	unsigned int i;
	if(wcharCp != NULL)
	return wcharCp;
	for(i=0; i< sizeof(wcharTries) / sizeof(wcharTries[0]); i++) {
	if(try(wcharTries[i]))
	return (wcharCp=wcharTries[i]);
	}
	fprintf(stderr, "No codepage found for wchar_t\n");
	return NULL;
	}


	doscp_t *cp_open(unsigned int codepage)
	{
	char dosCp[17];
	doscp_t *ret;
	iconv_t from;
	iconv_t to;

	if(codepage == 0)
	codepage = mtools_default_codepage;
	if(codepage > 9999) {
	fprintf(stderr, "Bad codepage %d\n", codepage);
	return NULL;
	}

	if(getWcharCp() == NULL)
	return NULL;

	sprintf(dosCp, "CP%d", codepage);
	from = iconv_open(wcharCp, dosCp);
	if(from == (iconv_t)-1) {
	fprintf(stderr, "Error converting to codepage %d %s\n",
	codepage, strerror(errno));
	return NULL;
	}

	sprintf(dosCp, "CP%d//TRANSLIT", codepage);
	to = iconv_open(dosCp, wcharCp);
	if(to == (iconv_t)-1) {
	/* Transliteration not supported? */
	sprintf(dosCp, "CP%d", codepage);
	to = iconv_open(dosCp, wcharCp);
	}
	if(to == (iconv_t)-1) {
	iconv_close(from);
	fprintf(stderr, "Error converting to codepage %d %s\n",
	codepage, strerror(errno));
	return NULL;
	}

	ret = New(doscp_t);
	if(ret == NULL)
	return ret;
	ret->from = from;
	ret->to = to;
	return ret;
	}

	void cp_close(doscp_t *cp)
	{
	iconv_close(cp->to);
	iconv_close(cp->from);
	free(cp);
	}

	size_t dos_to_wchar(doscp_t cp, const char dos, wchar_t *wchar, size_t len)
	{
	size_t r;
	size_t in_len=len;
	size_t out_len=len*sizeof(wchar_t);
	wchar_t *dptr=wchar;
	char dos2 = (char ) dos; /* Magic to be able to call iconv with its
	buggy prototype */
	r=iconv(cp->from, &dos2, &in_len, (char **)&dptr, &out_len);
	if(r == (size_t) -1)
	return r;
	*dptr = L'\0';
	return (size_t) (dptr-wchar);
	}

	/**
	* Converts len wide character to destination. Caller's responsibility to
	* ensure that dest is large enough.
	* mangled will be set if there has been an untranslatable character.
	*/
	static size_t safe_iconv(iconv_t conv, const wchar_t wchar, char dest,
	size_t in_len, size_t out_len, int *mangled)
	{
	size_t r;
	unsigned int i;
	char *dptr = dest;
	size_t len;

	in_len=in_len*sizeof(wchar_t);

	while(in_len > 0 && out_len > 0) {
	r=iconv(conv, (char**)&wchar, &in_len, &dptr, &out_len);
	if(r == (size_t) -1 \|\| errno != EILSEQ) {
	/* everything transformed, or error that is _not_ a bad
	* character */
	break;
	}
	*mangled \|= 1;

	if(out_len <= 0)
	break;
	if(dptr)
	*dptr++ = '_';
	in_len -= sizeof(wchar_t);

	wchar++;
	out_len--;
	}

	len = (size_t) (dptr-dest); /* how many dest characters have there been
	generated */

	/* eliminate question marks which might have been formed by
	untransliterable characters */
	for(i=0; i<len; i++) {
	if(dest[i] == '?') {
	dest[i] = '_';
	*mangled \|= 1;
	}
	}
	return len;
	}

	void wchar_to_dos(doscp_t *cp,
	wchar_t wchar, char dos, size_t len, int *mangled)
	{
	safe_iconv(cp->to, wchar, dos, len, len, mangled);
	}

	#else

	#include "codepage.h"

	struct doscp_t {
	unsigned char *from_dos;
	unsigned char to_dos[0x80];
	};

	doscp_t *cp_open(unsigned int codepage)
	{
	doscp_t *ret;
	int i;
	Codepage_t *cp;

	if(codepage == 0)
	codepage = 850;

	ret = New(doscp_t);
	if(ret == NULL)
	return ret;

	for(cp=codepages; cp->nr ; cp++)
	if(cp->nr == codepage) {
	ret->from_dos = cp->tounix;
	break;
	}

	if(ret->from_dos == NULL) {
	fprintf(stderr, "Bad codepage %d\n", codepage);
	free(ret);
	return NULL;
	}

	for(i=0; i<0x80; i++) {
	char native = ret->from_dos[i];
	if(! (native & 0x80))
	continue;
	ret->to_dos[native & 0x7f] = 0x80 \| i;
	}
	return ret;
	}

	void cp_close(doscp_t *cp)
	{
	free(cp);
	}

	size_t dos_to_wchar(doscp_t cp, const char dos, wchar_t *wchar, size_t len)
	{
	int i;

	for(i=0; i<len && dos[i]; i++) {
	char c = dos[i];
	if(c >= ' ' && c <= '~')
	wchar[i] = c;
	else {
	wchar[i] = cp->from_dos[c & 0x7f];
	}
	}
	wchar[i] = '\0';
	return i;
	}


	void wchar_to_dos(doscp_t *cp,
	wchar_t wchar, char dos, size_t len, int *mangled)
	{
	int i;
	for(i=0; i<len && wchar[i]; i++) {
	char c = wchar[i];
	if(c >= ' ' && c <= '~')
	dos[i] = c;
	else {
	dos[i] = cp->to_dos[c & 0x7f];
	if(dos[i] == '\0') {
	dos[i]='_';
	*mangled=1;
	}
	}
	}
	}

	#endif


	#ifndef HAVE_WCHAR_H

	typedef int mbstate_t;

	static inline size_t wcrtomb(char s, wchar_t wc, mbstate_t ps)
	{
	*s = wc;
	return 1;
	}

	static inline size_t mbrtowc(wchar_t pwc, const char s,
	size_t n, mbstate_t *ps)
	{
	pwc = s;
	return 1;
	}

	#endif

	#ifdef HAVE_ICONV_H

	#include <langinfo.h>

	static iconv_t to_native = NULL;

	static void initialize_to_native(void)
	{
	char li, cp;
	size_t len;
	if(to_native != NULL)
	return;
	li = nl_langinfo(CODESET);
	len = strlen(li) + 11;
	if(getWcharCp() == NULL)
	exit(1);
	cp = safe_malloc(len);
	strcpy(cp, li);
	strcat(cp, "//TRANSLIT");
	to_native = iconv_open(cp, wcharCp);
	if(to_native == (iconv_t) -1)
	to_native = iconv_open(li, wcharCp);
	if(to_native == (iconv_t) -1)
	fprintf(stderr, "Could not allocate iconv for %s\n", cp);
	free(cp);
	if(to_native == (iconv_t) -1)
	exit(1);
	}



	#endif


	/**
	* Convert wchar string to native, converting at most len wchar characters
	* Returns number of generated native characters
	*/
	size_t wchar_to_native(const wchar_t wchar, char native, size_t len,
	size_t out_len)
	{
	#ifdef HAVE_ICONV_H
	int mangled;
	size_t r;
	initialize_to_native();
	len = wcsnlen(wchar,len);
	r=safe_iconv(to_native, wchar, native, len, out_len, &mangled);
	native[r]='\0';
	return r;
	#else
	int i;
	char *dptr = native;
	mbstate_t ps;
	memset(&ps, 0, sizeof(ps));
	for(i=0; i<len && wchar[i] != 0; i++) {
	size_t r = wcrtomb(dptr, wchar[i], &ps);
	if(r == (size_t) -1 && errno == EILSEQ) {
	r=1;
	*dptr='_';
	}
	dptr+=r;
	}
	*dptr='\0';
	return dptr-native;
	#endif
	}

	/**
	* Convert native string to wchar string, generating at most len wchar
	* characters. If end is supplied, stop conversion when source pointer
	* exceeds end. Returns number of generated wchars
	*/
	size_t native_to_wchar(const char native, wchar_t wchar, size_t len,
	const char end, int mangled)
	{
	mbstate_t ps;
	unsigned int i;
	memset(&ps, 0, sizeof(ps));

	for(i=0; i<len && (native < end \|\| !end); i++) {
	size_t r = mbrtowc(wchar+i, native, len, &ps);
	if(r == (size_t) -1) {
	/* Unconvertible character. Just pretend it's Latin1
	encoded (if valid Latin1 character) or substitute
	with an underscore if not
	*/
	char c = *native;
	if(c >= '\xa0' && c < '\xff')
	wchar[i] = c & 0xff;
	else
	wchar[i] = '_';
	memset(&ps, 0, sizeof(ps));
	r=1;
	}
	if(r == 0)
	break;
	native += r;
	}
	if(mangled && ((end && native < end) \|\| (!end && *native && i == len)))
	*mangled \|= 3;
	wchar[i]='\0';
	return i;
	}