===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/utils/mb/wchar.c,v
retrieving revision 1.38
diff -c -r1.38 wchar.c
*** src/backend/utils/mb/wchar.c	17 Sep 2004 21:59:57 -0000	1.38
--- src/backend/utils/mb/wchar.c	21 Nov 2004 09:58:36 -0000
***************
*** 343,348 ****
--- 343,373 ----
  	return (pg_euc_dsplen(s));
  }
  
+ bool isLegalUTF8(const UTF8 *source, int len) {
+         UTF8 a;
+         const UTF8 *srcptr = source+len;
+         if(!source || (pg_utf_mblen(source) != len)) return false;
+         switch (len) {
+             default: return false;
+             /* Everything else falls through when "true"... */
+             case 6: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+             case 5: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+             case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+             case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
+             case 2: if ((a = (*--srcptr)) > 0xBF) return false;
+             switch (*source) {
+                     /* no fall-through in this inner switch */
+                     case 0xE0: if (a < 0xA0) return false; break;
+                     case 0xF0: if (a < 0x90) return false; break;
+                     case 0xF4: if (a > 0x8F) return false; break;
+                     default:  if (a < 0x80) return false;
+             }
+             case 1: if (*source >= 0x80 && *source < 0xC2) return false;
+             if (*source > 0xFD) return false;
+         }
+         return true;
+ }
+ 
  /*
   * convert UTF-8 string to pg_wchar (UCS-2)
   * caller should allocate enough space for "to"
***************
*** 398,404 ****
   * returns the byte length of a UTF-8 word pointed to by s
   */
  int
! pg_utf_mblen(const unsigned char *s)
  {
  	int			len = 1;
  
--- 423,429 ----
   * returns the byte length of a UTF-8 word pointed to by s
   */
  int
! pg_utf_mblen(const UTF8 *s)
  {
  	int			len = 1;
  
***************
*** 406,418 ****
  		len = 1;
  	else if ((*s & 0xe0) == 0xc0)
  		len = 2;
! 	else if ((*s & 0xe0) == 0xe0)
! 		len = 3;
  	return (len);
  }
  
  static int
! pg_utf_dsplen(const unsigned char *s)
  {
  	return 1;					/* XXX fix me! */
  }
--- 431,449 ----
  		len = 1;
  	else if ((*s & 0xe0) == 0xc0)
  		len = 2;
!         else if ((*s & 0xf0) == 0xe0)
!                 len = 3;
!         else if ((*s & 0xf8) == 0xf0)
!                 len = 4;
!         else if ((*s & 0xfc) == 0xf8)
!                 len = 5;
!         else if ((*s & 0xfe) == 0xfc)
!                 len = 6;
  	return (len);
  }
  
  static int
! pg_utf_dsplen(const UTF8 *s)
  {
  	return 1;					/* XXX fix me! */
  }
***************
*** 721,728 ****
  	{pg_euckr2wchar_with_len, pg_euckr_mblen, pg_euckr_dsplen, 3},		/* 3; PG_EUC_KR */
  	{pg_euctw2wchar_with_len, pg_euctw_mblen, pg_euctw_dsplen, 3},		/* 4; PG_EUC_TW */
  	{pg_johab2wchar_with_len, pg_johab_mblen, pg_johab_dsplen, 3},		/* 5; PG_JOHAB */
! 	{pg_utf2wchar_with_len, pg_utf_mblen, pg_utf_dsplen, 3},	/* 6; PG_UNICODE */
! 	{pg_mule2wchar_with_len, pg_mule_mblen, pg_mule_dsplen, 3}, /* 7; PG_MULE_INTERNAL */
  	{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, 1},	/* 8; PG_LATIN1 */
  	{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, 1},	/* 9; PG_LATIN2 */
  	{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, 1},	/* 10; PG_LATIN3 */
--- 752,759 ----
  	{pg_euckr2wchar_with_len, pg_euckr_mblen, pg_euckr_dsplen, 3},		/* 3; PG_EUC_KR */
  	{pg_euctw2wchar_with_len, pg_euctw_mblen, pg_euctw_dsplen, 3},		/* 4; PG_EUC_TW */
  	{pg_johab2wchar_with_len, pg_johab_mblen, pg_johab_dsplen, 3},		/* 5; PG_JOHAB */
! 	{pg_utf2wchar_with_len, pg_utf_mblen, pg_utf_dsplen, 6},		/* 6; PG_UNICODE */
! 	{pg_mule2wchar_with_len, pg_mule_mblen, pg_mule_dsplen, 3},		/* 7; PG_MULE_INTERNAL */
  	{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, 1},	/* 8; PG_LATIN1 */
  	{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, 1},	/* 9; PG_LATIN2 */
  	{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, 1},	/* 10; PG_LATIN3 */
***************
*** 744,754 ****
  	{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, 1},	/* 26; ISO-8859-7 */
  	{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, 1},	/* 27; ISO-8859-8 */
  	{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, 1},	/* 28; PG_WIN1250 */
! 	{0, pg_sjis_mblen, pg_sjis_dsplen, 2},		/* 29; PG_SJIS */
! 	{0, pg_big5_mblen, pg_big5_dsplen, 2},		/* 30; PG_BIG5 */
! 	{0, pg_gbk_mblen, pg_gbk_dsplen, 2},		/* 31; PG_GBK */
! 	{0, pg_uhc_mblen, pg_uhc_dsplen, 2},		/* 32; PG_UHC */
! 	{0, pg_gb18030_mblen, pg_gb18030_dsplen, 2} /* 33; PG_GB18030 */
  };
  
  /* returns the byte length of a word for mule internal code */
--- 775,785 ----
  	{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, 1},	/* 26; ISO-8859-7 */
  	{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, 1},	/* 27; ISO-8859-8 */
  	{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, 1},	/* 28; PG_WIN1250 */
! 	{0, pg_sjis_mblen, pg_sjis_dsplen, 2},					/* 29; PG_SJIS */
! 	{0, pg_big5_mblen, pg_big5_dsplen, 2},					/* 30; PG_BIG5 */
! 	{0, pg_gbk_mblen, pg_gbk_dsplen, 2},					/* 31; PG_GBK */
! 	{0, pg_uhc_mblen, pg_uhc_dsplen, 2},					/* 32; PG_UHC */
! 	{0, pg_gb18030_mblen, pg_gb18030_dsplen, 2}				/* 33; PG_GB18030 */
  };
  
  /* returns the byte length of a word for mule internal code */
***************
*** 822,872 ****
  
  	while (len > 0 && *mbstr)
  	{
- 		/* special UTF-8 check */
- 		if (encoding == PG_UTF8 && (*mbstr & 0xf8) == 0xf0)
- 		{
- 			if (noError)
- 				return false;
- 			ereport(ERROR,
- 					(errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
- 					 errmsg("Unicode characters greater than or equal to 0x10000 are not supported")));
- 		}
- 
  		l = pg_mblen(mbstr);
  
! 		for (i = 1; i < l; i++)
! 		{
! 			/*
! 			 * we expect that every multibyte char consists of bytes
! 			 * having the 8th bit set
! 			 */
! 			if (i >= len || (mbstr[i] & 0x80) == 0)
  			{
! 				char		buf[8 * 2 + 1];
! 				char	   *p = buf;
! 				int			j,
  							jlimit;
  
! 				if (noError)
! 					return false;
  
! 				jlimit = Min(l, len);
! 				jlimit = Min(jlimit, 8);		/* prevent buffer overrun */
  
! 				for (j = 0; j < jlimit; j++)
! 					p += sprintf(p, "%02x", mbstr[j]);
  
! 				ereport(ERROR,
! 						(errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
! 				errmsg("invalid byte sequence for encoding \"%s\": 0x%s",
! 					   GetDatabaseEncodingName(), buf)));
  			}
- 		}
  
  		len -= l;
  		mbstr += l;
  	}
- 
  	return true;
  }
  
--- 853,900 ----
  
  	while (len > 0 && *mbstr)
  	{
  		l = pg_mblen(mbstr);
  
! 		/* special UTF-8 check */
! 		if (encoding == PG_UTF8) {
! 			if(!isLegalUTF8(mbstr,l)) {
! 				if (noError) return false;
! 				ereport(ERROR,(errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),errmsg("Invalid UNICODE byte sequence detected near character %c",*mbstr)));
! 			}
! 		} else {
! 			for (i = 1; i < l; i++)
  			{
! 				/*
! 				 * we expect that every multibyte char consists of bytes
! 				 * having the 8th bit set
! 				 */
! 				if (i >= len || (mbstr[i] & 0x80) == 0)
! 				{
! 					char		buf[8 * 2 + 1];
! 					char	   *p = buf;
! 					int			j,
  							jlimit;
  
! 					if (noError)
! 						return false;
  
! 					jlimit = Min(l, len);
! 					jlimit = Min(jlimit, 8);		/* prevent buffer overrun */
  
! 					for (j = 0; j < jlimit; j++)
! 						p += sprintf(p, "%02x", mbstr[j]);
  
! 					ereport(ERROR,
! 							(errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
! 					errmsg("invalid byte sequence for encoding \"%s\": 0x%s",
! 						GetDatabaseEncodingName(), buf)));
! 				}
  			}
  
+ 		}
  		len -= l;
  		mbstr += l;
  	}
  	return true;
  }
  
Index: src/include/mb/pg_wchar.h
===================================================================
RCS file: /projects/cvsroot/pgsql/src/include/mb/pg_wchar.h,v
retrieving revision 1.52
diff -c -r1.52 pg_wchar.h
*** src/include/mb/pg_wchar.h	17 Sep 2004 21:59:57 -0000	1.52
--- src/include/mb/pg_wchar.h	21 Nov 2004 09:58:36 -0000
***************
*** 17,22 ****
--- 17,30 ----
   */
  typedef unsigned int pg_wchar;
  
+ 
+ /*
+  * The UTF types
+  */
+ typedef unsigned int	UTF32;  /* at least 32 bits */
+ typedef unsigned short	UTF16;  /* at least 16 bits */
+ typedef unsigned char	UTF8;   /* typically 8 bits */
+ 
  /*
   * various definitions for EUC
   */
***************
*** 339,342 ****
--- 347,352 ----
  extern void latin2mic_with_table(unsigned char *l, unsigned char *p, int len, int lc, unsigned char *tab);
  extern void mic2latin_with_table(unsigned char *mic, unsigned char *p, int len, int lc, unsigned char *tab);
  
+ extern bool isLegalUTF8(const UTF8 *source, int len);
+ 
  #endif   /* PG_WCHAR_H */