diff --git a/doc/src/sgml/ref/copy.sgml b/doc/src/sgml/ref/copy.sgml index 38424ad..6429a4e 100644 *** a/doc/src/sgml/ref/copy.sgml --- b/doc/src/sgml/ref/copy.sgml *************** COPY { ta *** 40,46 **** QUOTE 'quote_character' ESCAPE 'escape_character' FORCE_QUOTE { ( column [, ...] ) | * } ! FORCE_NOT_NULL ( column [, ...] ) --- 40,47 ---- QUOTE 'quote_character' ESCAPE 'escape_character' FORCE_QUOTE { ( column [, ...] ) | * } ! FORCE_NOT_NULL ( column [, ...] ) | ! ENCODING 'encoding_name' *************** COPY { ta *** 282,287 **** --- 283,300 ---- + + ENCODING + + + Specifies that the file is encoded in the encoding_name. If this option is + omitted, the current client encoding is used. See the Notes below + for more details. + + + + *************** COPY coun *** 377,384 **** ! Input data is interpreted according to the current client encoding, ! and output data is encoded in the current client encoding, even if the data does not pass through the client but is read from or written to a file directly by the server. --- 390,398 ---- ! Input data is interpreted according to ENCODING ! option or the current client encoding, and output data is encoded ! in ENCODING or the current client encoding, even if the data does not pass through the client but is read from or written to a file directly by the server. diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index 9f7263d..04bd71d 100644 *** a/src/backend/commands/copy.c --- b/src/backend/commands/copy.c *************** typedef struct CopyStateData *** 95,102 **** * dest == COPY_NEW_FE in COPY FROM */ bool fe_eof; /* true if detected end of copy data */ EolType eol_type; /* EOL type of input */ ! int client_encoding; /* remote side's character encoding */ ! bool need_transcoding; /* client encoding diff from server? */ bool encoding_embeds_ascii; /* ASCII can be non-first byte? */ /* parameters from the COPY command */ --- 95,102 ---- * dest == COPY_NEW_FE in COPY FROM */ bool fe_eof; /* true if detected end of copy data */ EolType eol_type; /* EOL type of input */ ! int file_encoding; /* file or remote side's character encoding */ ! bool need_transcoding; /* file encoding diff from server? */ bool encoding_embeds_ascii; /* ASCII can be non-first byte? */ /* parameters from the COPY command */ *************** typedef struct CopyStateData *** 110,116 **** bool header_line; /* CSV header line? */ char *null_print; /* NULL marker string (server encoding!) */ int null_print_len; /* length of same */ ! char *null_print_client; /* same converted to client encoding */ char *delim; /* column delimiter (must be 1 byte) */ char *quote; /* CSV quote char (must be 1 byte) */ char *escape; /* CSV escape char (must be 1 byte) */ --- 110,116 ---- bool header_line; /* CSV header line? */ char *null_print; /* NULL marker string (server encoding!) */ int null_print_len; /* length of same */ ! char *null_print_client; /* same converted to file encoding */ char *delim; /* column delimiter (must be 1 byte) */ char *quote; /* CSV quote char (must be 1 byte) */ char *escape; /* CSV escape char (must be 1 byte) */ *************** BeginCopy(bool is_from, *** 848,853 **** --- 848,854 ---- /* Allocate workspace and zero all fields */ cstate = (CopyStateData *) palloc0(sizeof(CopyStateData)); + cstate->file_encoding = -1; /* * We allocate everything used by a cstate in a new memory context. *************** BeginCopy(bool is_from, *** 964,969 **** --- 965,983 ---- errmsg("argument to option \"%s\" must be a list of column names", defel->defname))); } + else if (strcmp(defel->defname, "encoding") == 0) + { + if (cstate->file_encoding >= 0) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("conflicting or redundant options"))); + cstate->file_encoding = pg_char_to_encoding(defGetString(defel)); + if (cstate->file_encoding < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("argument to option \"%s\" must be a valid encoding name", + defel->defname))); + } else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), *************** BeginCopy(bool is_from, *** 1246,1262 **** } } /* ! * Set up encoding conversion info. Even if the client and server ! * encodings are the same, we must apply pg_client_to_server() to validate * data in multibyte encodings. */ - cstate->client_encoding = pg_get_client_encoding(); cstate->need_transcoding = ! (cstate->client_encoding != GetDatabaseEncoding() || pg_database_encoding_max_length() > 1); /* See Multibyte encoding comment above */ ! cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->client_encoding); cstate->copy_dest = COPY_FILE; /* default */ --- 1260,1279 ---- } } + /* Use client encoding when ENCODING option is not specified. */ + if (cstate->file_encoding < 0) + cstate->file_encoding = pg_get_client_encoding(); + /* ! * Set up encoding conversion info. Even if the file and server ! * encodings are the same, we must apply pg_any_to_server() to validate * data in multibyte encodings. */ cstate->need_transcoding = ! (cstate->file_encoding != GetDatabaseEncoding() || pg_database_encoding_max_length() > 1); /* See Multibyte encoding comment above */ ! cstate->encoding_embeds_ascii = PG_ENCODING_IS_CLIENT_ONLY(cstate->file_encoding); cstate->copy_dest = COPY_FILE; /* default */ *************** CopyTo(CopyState cstate) *** 1494,1505 **** else { /* ! * For non-binary copy, we need to convert null_print to client * encoding, because it will be sent directly with CopySendString. */ if (cstate->need_transcoding) ! cstate->null_print_client = pg_server_to_client(cstate->null_print, ! cstate->null_print_len); /* if a header has been requested send the line */ if (cstate->header_line) --- 1511,1523 ---- else { /* ! * For non-binary copy, we need to convert null_print to file * encoding, because it will be sent directly with CopySendString. */ if (cstate->need_transcoding) ! cstate->null_print_client = pg_server_to_any(cstate->null_print, ! cstate->null_print_len, ! cstate->file_encoding); /* if a header has been requested send the line */ if (cstate->header_line) *************** CopyReadLine(CopyState cstate) *** 2576,2583 **** { char *cvt; ! cvt = pg_client_to_server(cstate->line_buf.data, ! cstate->line_buf.len); if (cvt != cstate->line_buf.data) { /* transfer converted data back to line_buf */ --- 2594,2602 ---- { char *cvt; ! cvt = pg_any_to_server(cstate->line_buf.data, ! cstate->line_buf.len, ! cstate->file_encoding); if (cvt != cstate->line_buf.data) { /* transfer converted data back to line_buf */ *************** CopyReadLineText(CopyState cstate) *** 2822,2828 **** /* ----- * get next character * Note: we do not change c so if it isn't \., we can fall ! * through and continue processing for client encoding. * ----- */ c2 = copy_raw_buf[raw_buf_ptr]; --- 2841,2847 ---- /* ----- * get next character * Note: we do not change c so if it isn't \., we can fall ! * through and continue processing for file encoding. * ----- */ c2 = copy_raw_buf[raw_buf_ptr]; *************** not_end_of_copy: *** 2936,2942 **** mblen_str[0] = c; /* All our encodings only read the first byte to get the length */ ! mblen = pg_encoding_mblen(cstate->client_encoding, mblen_str); IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; --- 2955,2961 ---- mblen_str[0] = c; /* All our encodings only read the first byte to get the length */ ! mblen = pg_encoding_mblen(cstate->file_encoding, mblen_str); IF_NEED_REFILL_AND_NOT_EOF_CONTINUE(mblen - 1); IF_NEED_REFILL_AND_EOF_BREAK(mblen - 1); raw_buf_ptr += mblen - 1; *************** CopyAttributeOutText(CopyState cstate, c *** 3435,3441 **** char delimc = cstate->delim[0]; if (cstate->need_transcoding) ! ptr = pg_server_to_client(string, strlen(string)); else ptr = string; --- 3454,3460 ---- char delimc = cstate->delim[0]; if (cstate->need_transcoding) ! ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding); else ptr = string; *************** CopyAttributeOutText(CopyState cstate, c *** 3508,3514 **** start = ptr++; /* we include char in next run */ } else if (IS_HIGHBIT_SET(c)) ! ptr += pg_encoding_mblen(cstate->client_encoding, ptr); else ptr++; } --- 3527,3533 ---- start = ptr++; /* we include char in next run */ } else if (IS_HIGHBIT_SET(c)) ! ptr += pg_encoding_mblen(cstate->file_encoding, ptr); else ptr++; } *************** CopyAttributeOutCSV(CopyState cstate, ch *** 3595,3601 **** use_quote = true; if (cstate->need_transcoding) ! ptr = pg_server_to_client(string, strlen(string)); else ptr = string; --- 3614,3620 ---- use_quote = true; if (cstate->need_transcoding) ! ptr = pg_server_to_any(string, strlen(string), cstate->file_encoding); else ptr = string; *************** CopyAttributeOutCSV(CopyState cstate, ch *** 3622,3628 **** break; } if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii) ! tptr += pg_encoding_mblen(cstate->client_encoding, tptr); else tptr++; } --- 3641,3647 ---- break; } if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii) ! tptr += pg_encoding_mblen(cstate->file_encoding, tptr); else tptr++; } *************** CopyAttributeOutCSV(CopyState cstate, ch *** 3646,3652 **** start = ptr; /* we include char in next run */ } if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii) ! ptr += pg_encoding_mblen(cstate->client_encoding, ptr); else ptr++; } --- 3665,3671 ---- start = ptr; /* we include char in next run */ } if (IS_HIGHBIT_SET(c) && cstate->encoding_embeds_ascii) ! ptr += pg_encoding_mblen(cstate->file_encoding, ptr); else ptr++; } diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 3857205..320cde5 100644 *** a/src/backend/parser/gram.y --- b/src/backend/parser/gram.y *************** copy_opt_item: *** 2233,2238 **** --- 2233,2242 ---- { $$ = makeDefElem("force_not_null", (Node *)$4); } + | ENCODING Sconst + { + $$ = makeDefElem("encoding", (Node *)makeString($2)); + } ; /* The following exist for backward compatibility with very old versions */ diff --git a/src/backend/utils/mb/mbutils.c b/src/backend/utils/mb/mbutils.c index 5ee74f7..b8a2728 100644 *** a/src/backend/utils/mb/mbutils.c --- b/src/backend/utils/mb/mbutils.c *************** pg_encoding_max_length_sql(PG_FUNCTION_A *** 497,510 **** char * pg_client_to_server(const char *s, int len) { Assert(DatabaseEncoding); Assert(ClientEncoding); if (len <= 0) return (char *) s; ! if (ClientEncoding->encoding == DatabaseEncoding->encoding || ! ClientEncoding->encoding == PG_SQL_ASCII) { /* * No conversion is needed, but we must still validate the data. --- 497,521 ---- char * pg_client_to_server(const char *s, int len) { + Assert(ClientEncoding); + + return pg_any_to_server(s, len, ClientEncoding->encoding); + } + + /* + * convert any encoding to server encoding. + */ + char * + pg_any_to_server(const char *s, int len, int encoding) + { Assert(DatabaseEncoding); Assert(ClientEncoding); if (len <= 0) return (char *) s; ! if (encoding == DatabaseEncoding->encoding || ! encoding == PG_SQL_ASCII) { /* * No conversion is needed, but we must still validate the data. *************** pg_client_to_server(const char *s, int l *** 524,531 **** * to the parser but we have no way to convert it. We compromise by * rejecting the data if it contains any non-ASCII characters. */ ! if (PG_VALID_BE_ENCODING(ClientEncoding->encoding)) ! (void) pg_verify_mbstr(ClientEncoding->encoding, s, len, false); else { int i; --- 535,542 ---- * to the parser but we have no way to convert it. We compromise by * rejecting the data if it contains any non-ASCII characters. */ ! if (PG_VALID_BE_ENCODING(encoding)) ! (void) pg_verify_mbstr(encoding, s, len, false); else { int i; *************** pg_client_to_server(const char *s, int l *** 543,549 **** return (char *) s; } ! return perform_default_encoding_conversion(s, len, true); } /* --- 554,564 ---- return (char *) s; } ! if (ClientEncoding->encoding == encoding) ! return perform_default_encoding_conversion(s, len, true); ! else ! return (char *) pg_do_encoding_conversion( ! (unsigned char *) s, len, encoding, DatabaseEncoding->encoding); } /* *************** pg_client_to_server(const char *s, int l *** 552,569 **** char * pg_server_to_client(const char *s, int len) { Assert(DatabaseEncoding); Assert(ClientEncoding); if (len <= 0) return (char *) s; ! if (ClientEncoding->encoding == DatabaseEncoding->encoding || ! ClientEncoding->encoding == PG_SQL_ASCII || DatabaseEncoding->encoding == PG_SQL_ASCII) return (char *) s; /* assume data is valid */ ! return perform_default_encoding_conversion(s, len, false); } /* --- 567,599 ---- char * pg_server_to_client(const char *s, int len) { + Assert(ClientEncoding); + + return pg_any_to_server(s, len, ClientEncoding->encoding); + } + + /* + * convert server encoding to any encoding. + */ + char * + pg_server_to_any(const char *s, int len, int encoding) + { Assert(DatabaseEncoding); Assert(ClientEncoding); if (len <= 0) return (char *) s; ! if (encoding == DatabaseEncoding->encoding || ! encoding == PG_SQL_ASCII || DatabaseEncoding->encoding == PG_SQL_ASCII) return (char *) s; /* assume data is valid */ ! if (ClientEncoding->encoding == encoding) ! return perform_default_encoding_conversion(s, len, false); ! else ! return (char *) pg_do_encoding_conversion( ! (unsigned char *) s, len, DatabaseEncoding->encoding, encoding); } /* diff --git a/src/include/mb/pg_wchar.h b/src/include/mb/pg_wchar.h index 565b53b..85a7b2f 100644 *** a/src/include/mb/pg_wchar.h --- b/src/include/mb/pg_wchar.h *************** extern unsigned char *pg_do_encoding_con *** 420,425 **** --- 420,427 ---- extern char *pg_client_to_server(const char *s, int len); extern char *pg_server_to_client(const char *s, int len); + extern char *pg_any_to_server(const char *s, int len, int encoding); + extern char *pg_server_to_any(const char *s, int len, int encoding); extern unsigned short BIG5toCNS(unsigned short big5, unsigned char *lc); extern unsigned short CNStoBIG5(unsigned short cns, unsigned char lc); diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out index 15cbe02..8e2bc0c 100644 *** a/src/test/regress/expected/copy2.out --- b/src/test/regress/expected/copy2.out *************** CONTEXT: COPY x, line 1: "2001 231 \N \ *** 46,55 **** COPY x from stdin; ERROR: extra data after last expected column CONTEXT: COPY x, line 1: "2002 232 40 50 60 70 80" ! -- various COPY options: delimiters, oids, NULL string COPY x (b, c, d, e) from stdin with oids delimiter ',' null 'x'; COPY x from stdin WITH DELIMITER AS ';' NULL AS ''; ! COPY x from stdin WITH DELIMITER AS ':' NULL AS E'\\X'; -- check results of copy in SELECT * FROM x; a | b | c | d | e --- 46,55 ---- COPY x from stdin; ERROR: extra data after last expected column CONTEXT: COPY x, line 1: "2002 232 40 50 60 70 80" ! -- various COPY options: delimiters, oids, NULL string, encoding COPY x (b, c, d, e) from stdin with oids delimiter ',' null 'x'; COPY x from stdin WITH DELIMITER AS ';' NULL AS ''; ! COPY x from stdin WITH DELIMITER AS ':' NULL AS E'\\X' ENCODING 'sql_ascii'; -- check results of copy in SELECT * FROM x; a | b | c | d | e *************** COPY y TO stdout WITH CSV QUOTE '''' DEL *** 187,193 **** Jackson, Sam|\h It is "perfect".| ''| ! COPY y TO stdout WITH CSV FORCE QUOTE col2 ESCAPE E'\\'; "Jackson, Sam","\\h" "It is \"perfect\"."," " "", --- 187,193 ---- Jackson, Sam|\h It is "perfect".| ''| ! COPY y TO stdout WITH CSV FORCE QUOTE col2 ESCAPE E'\\' ENCODING 'sql_ascii'; "Jackson, Sam","\\h" "It is \"perfect\"."," " "", diff --git a/src/test/regress/sql/copy2.sql b/src/test/regress/sql/copy2.sql index c2e8b03..6322c8f 100644 *** a/src/test/regress/sql/copy2.sql --- b/src/test/regress/sql/copy2.sql *************** COPY x from stdin; *** 72,78 **** 2002 232 40 50 60 70 80 \. ! -- various COPY options: delimiters, oids, NULL string COPY x (b, c, d, e) from stdin with oids delimiter ',' null 'x'; 500000,x,45,80,90 500001,x,\x,\\x,\\\x --- 72,78 ---- 2002 232 40 50 60 70 80 \. ! -- various COPY options: delimiters, oids, NULL string, encoding COPY x (b, c, d, e) from stdin with oids delimiter ',' null 'x'; 500000,x,45,80,90 500001,x,\x,\\x,\\\x *************** COPY x from stdin WITH DELIMITER AS ';' *** 83,89 **** 3000;;c;; \. ! COPY x from stdin WITH DELIMITER AS ':' NULL AS E'\\X'; 4000:\X:C:\X:\X 4001:1:empty:: 4002:2:null:\X:\X --- 83,89 ---- 3000;;c;; \. ! COPY x from stdin WITH DELIMITER AS ':' NULL AS E'\\X' ENCODING 'sql_ascii'; 4000:\X:C:\X:\X 4001:1:empty:: 4002:2:null:\X:\X *************** INSERT INTO y VALUES ('', NULL); *** 127,133 **** COPY y TO stdout WITH CSV; COPY y TO stdout WITH CSV QUOTE '''' DELIMITER '|'; ! COPY y TO stdout WITH CSV FORCE QUOTE col2 ESCAPE E'\\'; COPY y TO stdout WITH CSV FORCE QUOTE *; -- Repeat above tests with new 9.0 option syntax --- 127,133 ---- COPY y TO stdout WITH CSV; COPY y TO stdout WITH CSV QUOTE '''' DELIMITER '|'; ! COPY y TO stdout WITH CSV FORCE QUOTE col2 ESCAPE E'\\' ENCODING 'sql_ascii'; COPY y TO stdout WITH CSV FORCE QUOTE *; -- Repeat above tests with new 9.0 option syntax