Index: src/backend/commands/copy.c
===================================================================
RCS file: /projects/cvsroot/pgsql/src/backend/commands/copy.c,v
retrieving revision 1.247
diff -c -r1.247 copy.c
*** src/backend/commands/copy.c	10 Jul 2005 21:13:58 -0000	1.247
--- src/backend/commands/copy.c	15 Jul 2005 00:07:49 -0000
***************
*** 8,14 ****
   *
   *
   * IDENTIFICATION
!  *	  $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.247 2005/07/10 21:13:58 tgl Exp $
   *
   *-------------------------------------------------------------------------
   */
--- 8,14 ----
   *
   *
   * IDENTIFICATION
!  *	  $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.246 2005/06/28 05:08:53 tgl Exp $
   *
   *-------------------------------------------------------------------------
   */
***************
*** 50,55 ****
--- 50,56 ----
  
  #define ISOCTAL(c) (((c) >= '0') && ((c) <= '7'))
  #define OCTVALUE(c) ((c) - '0')
+ #define COPY_BUF_SIZE 65536
  
  /*
   * Represents the different source/dest cases we need to worry about at
***************
*** 63,78 ****
  } CopyDest;
  
  /*
-  * State indicator showing what stopped CopyReadAttribute()
-  */
- typedef enum CopyReadResult
- {
- 	NORMAL_ATTR,
- 	END_OF_LINE,
- 	UNTERMINATED_FIELD
- } CopyReadResult;
- 
- /*
   *	Represents the end-of-line terminator type of the input
   */
  typedef enum EolType
--- 64,69 ----
***************
*** 97,102 ****
--- 88,96 ----
  static EolType eol_type;		/* EOL type of input */
  static int	client_encoding;	/* remote side's character encoding */
  static int	server_encoding;	/* local encoding */
+ static char	eol_ch[2];			/* The byte values of the 1 or 2 eol bytes */
+ static bool client_encoding_only; /* true if client encoding is a non
+                                    * supported server encoding */
  
  /* these are just for error messages, see copy_in_error_callback */
  static bool copy_binary;		/* is it a binary copy? */
***************
*** 104,126 ****
  static int	copy_lineno;		/* line number for error messages */
  static const char *copy_attname;	/* current att for error messages */
  
  
  /*
   * These static variables are used to avoid incurring overhead for each
!  * attribute processed.  attribute_buf is reused on each CopyReadAttribute
   * call to hold the string being read in.  Under normal use it will soon
   * grow to a suitable size, and then we will avoid palloc/pfree overhead
   * for subsequent attributes.  Note that CopyReadAttribute returns a pointer
!  * to attribute_buf's data buffer!
   */
! static StringInfoData attribute_buf;
  
  /*
   * Similarly, line_buf holds the whole input line being processed (its
   * cursor field points to the next character to be read by CopyReadAttribute).
   * The input cycle is first to read the whole line into line_buf, convert it
   * to server encoding, and then extract individual attribute fields into
!  * attribute_buf.  (We used to have CopyReadAttribute read the input source
   * directly, but that caused a lot of encoding issues and unnecessary logic
   * complexity.)
   */
--- 98,141 ----
  static int	copy_lineno;		/* line number for error messages */
  static const char *copy_attname;	/* current att for error messages */
  
+ /*
+  * Static variables for buffered input parsing 
+  */
+ static char input_buf[COPY_BUF_SIZE + 1]; /* extra byte for '\0' */
+ static int	buffer_index;		/* input buffer index */
+ static bool end_marker;
+ static char *begloc;
+ static char *endloc;
+ static bool buf_done;			/* finished processing the current buffer */
+ static bool line_done;			/* finished processing the whole line or
+                                  * stopped in the middle */
+ /* these are for CSV format */
+ static bool	in_quote;
+ static bool last_was_esc;
+ 
+ /* these are for TEXT format */
+ static bool esc_in_prevbuf;		/* escape was last character of the data
+                                  * input buffer */
+ static bool cr_in_prevbuf;		/* CR was last character of the data input
+                                  * buffer */
+ 
  
  /*
   * These static variables are used to avoid incurring overhead for each
!  * attribute processed.  attr_buf is reused on each CopyReadAttribute
   * call to hold the string being read in.  Under normal use it will soon
   * grow to a suitable size, and then we will avoid palloc/pfree overhead
   * for subsequent attributes.  Note that CopyReadAttribute returns a pointer
!  * to attr_buf's data buffer!
   */
! static StringInfoData attr_buf; 
  
  /*
   * Similarly, line_buf holds the whole input line being processed (its
   * cursor field points to the next character to be read by CopyReadAttribute).
   * The input cycle is first to read the whole line into line_buf, convert it
   * to server encoding, and then extract individual attribute fields into
!  * attr_buf.  (We used to have CopyReadAttribute read the input source
   * directly, but that caused a lot of encoding issues and unnecessary logic
   * complexity.)
   */
***************
*** 137,155 ****
  static void CopyFrom(Relation rel, List *attnumlist, bool binary, bool oids,
   char *delim, char *null_print, bool csv_mode, char *quote, char *escape,
  		 List *force_notnull_atts, bool header_line);
! static bool CopyReadLine(char * quote, char * escape);
! static char *CopyReadAttribute(const char *delim, const char *null_print,
! 				  CopyReadResult *result, bool *isnull);
! static char *CopyReadAttributeCSV(const char *delim, const char *null_print,
! 					 char *quote, char *escape,
! 					 CopyReadResult *result, bool *isnull);
  static Datum CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo,
! 						Oid typioparam, int32 typmod, bool *isnull);
  static void CopyAttributeOut(char *string, char *delim);
  static void CopyAttributeOutCSV(char *string, char *delim, char *quote,
  					char *escape, bool force_quote);
  static List *CopyGetAttnums(Relation rel, List *attnamelist);
  static void limit_printout_length(StringInfo buf);
  
  /* Internal communications functions */
  static void SendCopyBegin(bool binary, int natts);
--- 152,176 ----
  static void CopyFrom(Relation rel, List *attnumlist, bool binary, bool oids,
   char *delim, char *null_print, bool csv_mode, char *quote, char *escape,
  		 List *force_notnull_atts, bool header_line);
! static bool CopyReadLineText(size_t bytesread, char *escape);
! static bool CopyReadLineCSV(size_t bytesread, char *quote, char *escape);
! static void CopyReadAttributesText(const char *delim, const char *escape, const char *null_print,
! 								   int null_print_len, char *nulls, List *attnumlist, 
! 								   int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr);
! static void CopyReadAttributesCSV(const char *delim, const char *null_print, char *quote,
! 								  char *escape, int null_print_len, char *nulls, List *attnumlist, 
! 								  int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr);
  static Datum CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo,
! 						Oid typioparam, bool *isnull);
! static char *CopyReadOidAttr(const char *delim, const char *null_print, int null_print_len,
! 				             bool *isnull);
  static void CopyAttributeOut(char *string, char *delim);
  static void CopyAttributeOutCSV(char *string, char *delim, char *quote,
  					char *escape, bool force_quote);
  static List *CopyGetAttnums(Relation rel, List *attnamelist);
  static void limit_printout_length(StringInfo buf);
+ static bool DetectLineEnd(size_t bytesread, char *quote, char *escape);
+ 
  
  /* Internal communications functions */
  static void SendCopyBegin(bool binary, int natts);
***************
*** 159,175 ****
  static void CopySendString(const char *str);
  static void CopySendChar(char c);
  static void CopySendEndOfRow(bool binary);
! static void CopyGetData(void *databuf, int datasize);
! static int	CopyGetChar(void);
  
  #define CopyGetEof()  (fe_eof)
- static int	CopyPeekChar(void);
- static void CopyDonePeek(int c, bool pickup);
  static void CopySendInt32(int32 val);
  static int32 CopyGetInt32(void);
  static void CopySendInt16(int16 val);
  static int16 CopyGetInt16(void);
  
  
  /*
   * Send copy start/stop messages for frontend copies.  These have changed
--- 180,197 ----
  static void CopySendString(const char *str);
  static void CopySendChar(char c);
  static void CopySendEndOfRow(bool binary);
! static int	CopyGetData(void *databuf, int datasize);
  
  #define CopyGetEof()  (fe_eof)
  static void CopySendInt32(int32 val);
  static int32 CopyGetInt32(void);
  static void CopySendInt16(int16 val);
  static int16 CopyGetInt16(void);
  
+ /* byte scaning utils */
+ static char *scanTextLine(const char *s, char c, size_t len);
+ static char *scanCSVLine(const char *s, char c1, char c2, char c3, size_t len);
+ static char *scanTextAttr(const char *s, char c1, char c2, size_t len);
  
  /*
   * Send copy start/stop messages for frontend copies.  These have changed
***************
*** 382,395 ****
   * It seems unwise to allow the COPY IN to complete normally in that case.
   *
   * NB: no data conversion is applied by these functions
   */
! static void
  CopyGetData(void *databuf, int datasize)
  {
  	switch (copy_dest)
  	{
  		case COPY_FILE:
! 			fread(databuf, datasize, 1, copy_file);
  			if (feof(copy_file))
  				fe_eof = true;
  			break;
--- 404,422 ----
   * It seems unwise to allow the COPY IN to complete normally in that case.
   *
   * NB: no data conversion is applied by these functions
+  *
+  * Returns: the number of bytes that were successfully read
+  * into the data buffer.
   */
! static int
  CopyGetData(void *databuf, int datasize)
  {
+ 	size_t		bytesread = 0;
+ 
  	switch (copy_dest)
  	{
  		case COPY_FILE:
! 			bytesread = fread(databuf, 1, datasize, copy_file);
  			if (feof(copy_file))
  				fe_eof = true;
  			break;
***************
*** 401,406 ****
--- 428,435 ----
  						(errcode(ERRCODE_CONNECTION_FAILURE),
  						 errmsg("unexpected EOF on client connection")));
  			}
+ 			bytesread += datasize;		/* update the count of bytes that were
+ 										 * read so far */
  			break;
  		case COPY_NEW_FE:
  			while (datasize > 0 && !fe_eof)
***************
*** 429,435 ****
  						case 'c':		/* CopyDone */
  							/* COPY IN correctly terminated by frontend */
  							fe_eof = true;
! 							return;
  						case 'f':		/* CopyFail */
  							ereport(ERROR,
  									(errcode(ERRCODE_QUERY_CANCELED),
--- 458,464 ----
  						case 'c':		/* CopyDone */
  							/* COPY IN correctly terminated by frontend */
  							fe_eof = true;
! 							return bytesread;
  						case 'f':		/* CopyFail */
  							ereport(ERROR,
  									(errcode(ERRCODE_QUERY_CANCELED),
***************
*** 459,598 ****
  					avail = datasize;
  				pq_copymsgbytes(copy_msgbuf, databuf, avail);
  				databuf = (void *) ((char *) databuf + avail);
  				datasize -= avail;
  			}
  			break;
  	}
- }
- 
- static int
- CopyGetChar(void)
- {
- 	int			ch;
- 
- 	switch (copy_dest)
- 	{
- 		case COPY_FILE:
- 			ch = getc(copy_file);
- 			break;
- 		case COPY_OLD_FE:
- 			ch = pq_getbyte();
- 			if (ch == EOF)
- 			{
- 				/* Only a \. terminator is legal EOF in old protocol */
- 				ereport(ERROR,
- 						(errcode(ERRCODE_CONNECTION_FAILURE),
- 						 errmsg("unexpected EOF on client connection")));
- 			}
- 			break;
- 		case COPY_NEW_FE:
- 			{
- 				unsigned char cc;
- 
- 				CopyGetData(&cc, 1);
- 				if (fe_eof)
- 					ch = EOF;
- 				else
- 					ch = cc;
- 				break;
- 			}
- 		default:
- 			ch = EOF;
- 			break;
- 	}
- 	if (ch == EOF)
- 		fe_eof = true;
- 	return ch;
- }
- 
- /*
-  * CopyPeekChar reads a byte in "peekable" mode.
-  *
-  * after each call to CopyPeekChar, a call to CopyDonePeek _must_
-  * follow, unless EOF was returned.
-  *
-  * CopyDonePeek will either take the peeked char off the stream
-  * (if pickup is true) or leave it on the stream (if pickup is false).
-  */
- static int
- CopyPeekChar(void)
- {
- 	int			ch;
- 
- 	switch (copy_dest)
- 	{
- 		case COPY_FILE:
- 			ch = getc(copy_file);
- 			break;
- 		case COPY_OLD_FE:
- 			ch = pq_peekbyte();
- 			if (ch == EOF)
- 			{
- 				/* Only a \. terminator is legal EOF in old protocol */
- 				ereport(ERROR,
- 						(errcode(ERRCODE_CONNECTION_FAILURE),
- 						 errmsg("unexpected EOF on client connection")));
- 			}
- 			break;
- 		case COPY_NEW_FE:
- 			{
- 				unsigned char cc;
- 
- 				CopyGetData(&cc, 1);
- 				if (fe_eof)
- 					ch = EOF;
- 				else
- 					ch = cc;
- 				break;
- 			}
- 		default:
- 			ch = EOF;
- 			break;
- 	}
- 	if (ch == EOF)
- 		fe_eof = true;
- 	return ch;
- }
- 
- static void
- CopyDonePeek(int c, bool pickup)
- {
- 	if (fe_eof)
- 		return;					/* can't unget an EOF */
- 	switch (copy_dest)
- 	{
- 		case COPY_FILE:
- 			if (!pickup)
- 			{
- 				/* We don't want to pick it up - so put it back in there */
- 				ungetc(c, copy_file);
- 			}
- 			/* If we wanted to pick it up, it's already done */
- 			break;
- 		case COPY_OLD_FE:
- 			if (pickup)
- 			{
- 				/* We want to pick it up */
- 				(void) pq_getbyte();
- 			}
  
! 			/*
! 			 * If we didn't want to pick it up, just leave it where it
! 			 * sits
! 			 */
! 			break;
! 		case COPY_NEW_FE:
! 			if (!pickup)
! 			{
! 				/* We don't want to pick it up - so put it back in there */
! 				copy_msgbuf->cursor--;
! 			}
! 			/* If we wanted to pick it up, it's already done */
! 			break;
! 	}
  }
  
- 
  /*
   * These functions do apply some data conversion
   */
--- 488,503 ----
  					avail = datasize;
  				pq_copymsgbytes(copy_msgbuf, databuf, avail);
  				databuf = (void *) ((char *) databuf + avail);
+ 				bytesread += avail;		/* update the count of bytes that were
+ 										 * read so far */
  				datasize -= avail;
  			}
  			break;
  	}
  
! 	return bytesread;
  }
  
  /*
   * These functions do apply some data conversion
   */
***************
*** 968,980 ****
  	}
  
  	/* Set up variables to avoid per-attribute overhead. */
! 	initStringInfo(&attribute_buf);
  	initStringInfo(&line_buf);
  	line_buf_converted = false;
  
  	client_encoding = pg_get_client_encoding();
  	server_encoding = GetDatabaseEncoding();
  
  	copy_dest = COPY_FILE;		/* default */
  	copy_file = NULL;
  	copy_msgbuf = NULL;
--- 873,905 ----
  	}
  
  	/* Set up variables to avoid per-attribute overhead. */
! 	initStringInfo(&attr_buf);
  	initStringInfo(&line_buf);
  	line_buf_converted = false;
  
  	client_encoding = pg_get_client_encoding();
  	server_encoding = GetDatabaseEncoding();
  
+ 	/*
+ 	 * check if the client encoding is one of the 5 encodings
+ 	 * that are not supported as a server encodings.
+ 	 */
+ 	switch (client_encoding)
+ 	{
+ 		case PG_SJIS:
+ 		case PG_BIG5:
+ 		case PG_GBK:
+ 		case PG_UHC:
+ 		case PG_GB18030:
+ 			client_encoding_only = true;
+ 			break;
+ 		default:
+ 			client_encoding_only = false;
+ 	}
+ 
+ 	if(!csv_mode)
+ 	    escape = "\\";		/* default for text format */
+ 
  	copy_dest = COPY_FILE;		/* default */
  	copy_file = NULL;
  	copy_msgbuf = NULL;
***************
*** 1105,1113 ****
  					 errmsg("could not write to file \"%s\": %m",
  							filename)));
  	}
! 	pfree(attribute_buf.data);
  	pfree(line_buf.data);
  
  	/*
  	 * Close the relation.	If reading, we can release the AccessShareLock
  	 * we got; if writing, we should hold the lock until end of
--- 1030,1039 ----
  					 errmsg("could not write to file \"%s\": %m",
  							filename)));
  	}
! 	pfree(attr_buf.data);
  	pfree(line_buf.data);
  
+ 
  	/*
  	 * Close the relation.	If reading, we can release the AccessShareLock
  	 * we got; if writing, we should hold the lock until end of
***************
*** 1345,1351 ****
  				{
  					bytea	   *outputbytes;
  
! 					outputbytes = DatumGetByteaP(FunctionCall1(&out_functions[attnum - 1],
  															   value));
  					/* We assume the result will not have been toasted */
  					CopySendInt32(VARSIZE(outputbytes) - VARHDRSZ);
--- 1271,1278 ----
  				{
  					bytea	   *outputbytes;
  
! 					outputbytes =
! 						DatumGetByteaP(FunctionCall1(&out_functions[attnum - 1],
  															   value));
  					/* We assume the result will not have been toasted */
  					CopySendInt32(VARSIZE(outputbytes) - VARHDRSZ);
***************
*** 1395,1404 ****
  		if (copy_attname)
  		{
  			/* error is relevant to a particular column */
! 			limit_printout_length(&attribute_buf);
  			errcontext("COPY %s, line %d, column %s: \"%s\"",
  					   copy_relname, copy_lineno, copy_attname,
! 					   attribute_buf.data);
  		}
  		else
  		{
--- 1322,1331 ----
  		if (copy_attname)
  		{
  			/* error is relevant to a particular column */
! 			limit_printout_length(&attr_buf);
  			errcontext("COPY %s, line %d, column %s: \"%s\"",
  					   copy_relname, copy_lineno, copy_attname,
! 					   attr_buf.data);
  		}
  		else
  		{
***************
*** 1406,1411 ****
--- 1333,1340 ----
  			if (line_buf_converted ||
  				client_encoding == server_encoding)
  			{
+ 				/* Strip off the newline */
+ 				*(line_buf.data + line_buf.len - 1) = '\0';
  				limit_printout_length(&line_buf);
  				errcontext("COPY %s, line %d: \"%s\"",
  						   copy_relname, copy_lineno,
***************
*** 1485,1491 ****
  	Oid			in_func_oid;
  	Datum	   *values;
  	char	   *nulls;
- 	bool		done = false;
  	bool		isnull;
  	ResultRelInfo *resultRelInfo;
  	EState	   *estate = CreateExecutorState(); /* for ExecConstraints() */
--- 1414,1419 ----
***************
*** 1496,1501 ****
--- 1424,1434 ----
  	ExprContext *econtext;		/* used for ExecEvalExpr for default atts */
  	MemoryContext oldcontext = CurrentMemoryContext;
  	ErrorContextCallback errcontext;
+     int		   *attr_offsets;
+ 	int			null_print_len; /* length of null print */
+ 	bool		no_more_data;
+ 	ListCell   *cur;
+ 
  
  	tupDesc = RelationGetDescr(rel);
  	attr = tupDesc->attrs;
***************
*** 1607,1613 ****
  		}
  	}
  
! 	/* Prepare to catch AFTER triggers. */
  	AfterTriggerBeginQuery();
  
  	/*
--- 1540,1548 ----
  		}
  	}
  
! 	/*
! 	 * Prepare to catch AFTER triggers.
! 	 */
  	AfterTriggerBeginQuery();
  
  	/*
***************
*** 1671,1676 ****
--- 1606,1612 ----
  
  	values = (Datum *) palloc(num_phys_attrs * sizeof(Datum));
  	nulls = (char *) palloc(num_phys_attrs * sizeof(char));
+ 	attr_offsets = (int *) palloc(num_phys_attrs * sizeof(int));
  
  	/* Make room for a PARAM_EXEC value for domain constraint checks */
  	if (hasConstraints)
***************
*** 1691,1712 ****
  	errcontext.previous = error_context_stack;
  	error_context_stack = &errcontext;
  
! 	/* on input just throw the header line away */
! 	if (header_line)
  	{
! 		copy_lineno++;
! 		done = CopyReadLine(quote, escape) ;
  	}
  
! 	while (!done)
  	{
  		bool		skip_tuple;
  		Oid			loaded_oid = InvalidOid;
  
  		CHECK_FOR_INTERRUPTS();
  
- 		copy_lineno++;
- 
  		/* Reset the per-tuple exprcontext */
  		ResetPerTupleExprContext(estate);
  
--- 1627,1681 ----
  	errcontext.previous = error_context_stack;
  	error_context_stack = &errcontext;
  
! 	/*
! 	 * initialize buffered scan variables.
! 	 */
! 	if(csv_mode)
! 	{
! 	    in_quote = false;
! 		last_was_esc = false;
! 	}
! 
! 	null_print_len = strlen(null_print);
! 	
! 	/* Set up data buffer to hold a chunk of data */
! 	MemSet(input_buf, ' ', COPY_BUF_SIZE * sizeof(char));
! 	input_buf[COPY_BUF_SIZE] = '\0';
! 
! 	no_more_data = false;		/* no more input data to read from file or FE */
! 	line_done = true;
! 	buf_done = false;
! 
! 	do
  	{
! 		size_t	   bytesread = 0;
! 		
! 		/* read a chunk of data into the buffer */
! 		if (!binary)
! 		{
! 		   bytesread = CopyGetData(input_buf, COPY_BUF_SIZE);
! 		   buf_done = false;
! 
! 		   /* set buffer pointers to beginning of the buffer */
! 		   begloc = input_buf;
! 		   buffer_index = 0;
  	}
  
! 		/*
! 		 * continue if some bytes were read or if we didn't reach EOF. if we
! 		 * both reached EOF _and_ no bytes were read, quit the loop we are
! 		 * done
! 		 */
! 		if (bytesread > 0 || !fe_eof || binary)
! 		{
! 
! 			while (!buf_done)
  	{
  		bool		skip_tuple;
  		Oid			loaded_oid = InvalidOid;
  
  		CHECK_FOR_INTERRUPTS();
  
  		/* Reset the per-tuple exprcontext */
  		ResetPerTupleExprContext(estate);
  
***************
*** 1716,1746 ****
  		/* Initialize all values for row to NULL */
  		MemSet(values, 0, num_phys_attrs * sizeof(Datum));
  		MemSet(nulls, 'n', num_phys_attrs * sizeof(char));
  
  		if (!binary)
  		{
- 			CopyReadResult result = NORMAL_ATTR;
- 			char	   *string;
- 			ListCell   *cur;
  
  			/* Actually read the line into memory here */
! 			done = csv_mode ? 
! 				CopyReadLine(quote, escape) : CopyReadLine(NULL, NULL);
  
  			/*
! 			 * EOF at start of line means we're done.  If we see EOF after
! 			 * some characters, we act as though it was newline followed
! 			 * by EOF, ie, process the line and then exit loop on next
! 			 * iteration.
  			 */
! 			if (done && line_buf.len == 0)
  				break;
  
  			if (file_has_oids)
  			{
  				/* can't be in CSV mode here */
! 				string = CopyReadAttribute(delim, null_print,
! 										   &result, &isnull);
  
  				if (isnull)
  					ereport(ERROR,
--- 1685,1729 ----
  		/* Initialize all values for row to NULL */
  		MemSet(values, 0, num_phys_attrs * sizeof(Datum));
  		MemSet(nulls, 'n', num_phys_attrs * sizeof(char));
+ 				/* reset attribute pointers */
+ 				MemSet(attr_offsets, 0, num_phys_attrs * sizeof(int));
  
  		if (!binary)
  		{
  
  			/* Actually read the line into memory here */
! 					line_done = csv_mode ?
! 								CopyReadLineCSV(bytesread, quote, escape) : 
! 								CopyReadLineText(bytesread, escape);
  
  			/*
! 					 * if finished processing data line - increment line count.
! 					 * Otherwise, if eof is not yet reached, we skip att parsing 
! 					 * and read more data. But if eof _was_ reached it means 
! 					 * that the original last data line is defective and 
! 					 * we want to catch that error later on.
  			 */
! 					if (line_done)
! 						copy_lineno++;
! 					else if (!fe_eof || end_marker )
  				break;
  
+ 					if (header_line)
+ 					{
+ 						line_buf.len = 0;		/* we can reset line buffer now. */
+ 						line_buf.data[0] = '\0';
+ 						line_buf.cursor = 0;
+ 						header_line = false;
+ 						continue;
+ 					}
+ 					
  			if (file_has_oids)
  			{
+ 						char	   *oid_string;
+ 
  				/* can't be in CSV mode here */
! 						oid_string = CopyReadOidAttr(delim, null_print, null_print_len,
! 													 &isnull);
  
  				if (isnull)
  					ereport(ERROR,
***************
*** 1750,1756 ****
  				{
  					copy_attname = "oid";
  					loaded_oid = DatumGetObjectId(DirectFunctionCall1(oidin,
! 											   CStringGetDatum(string)));
  					if (loaded_oid == InvalidOid)
  						ereport(ERROR,
  								(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
--- 1733,1739 ----
  				{
  					copy_attname = "oid";
  					loaded_oid = DatumGetObjectId(DirectFunctionCall1(oidin,
! 												   CStringGetDatum(oid_string)));
  					if (loaded_oid == InvalidOid)
  						ereport(ERROR,
  								(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
***************
*** 1759,1792 ****
  				}
  			}
  
! 			/* Loop to read the user attributes on the line. */
  			foreach(cur, attnumlist)
  			{
  				int			attnum = lfirst_int(cur);
  				int			m = attnum - 1;
  
! 				/*
! 				 * If prior attr on this line was ended by newline,
! 				 * complain.
! 				 */
! 				if (result != NORMAL_ATTR)
! 					ereport(ERROR,
! 							(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 							 errmsg("missing data for column \"%s\"",
! 									NameStr(attr[m]->attname))));
  
! 				if (csv_mode)
! 				{
! 					string = CopyReadAttributeCSV(delim, null_print, quote,
! 											   escape, &result, &isnull);
! 					if (result == UNTERMINATED_FIELD)
! 						ereport(ERROR,
! 								(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 							   errmsg("unterminated CSV quoted field")));
! 				}
  				else
! 					string = CopyReadAttribute(delim, null_print,
! 											   &result, &isnull);
  
  				if (csv_mode && isnull && force_notnull[m])
  				{
--- 1742,1770 ----
  				}
  			}
  
! 					/* parse all the attribute in the data line */
! 					if(csv_mode)
! 						CopyReadAttributesCSV(delim, null_print, quote, escape, null_print_len, 
! 											nulls, attnumlist, attr_offsets, num_phys_attrs, attr);
! 					else
! 						CopyReadAttributesText(delim, escape, null_print, null_print_len,
! 											 nulls, attnumlist, attr_offsets, num_phys_attrs, attr);
! 
! 					/*
! 					 * Loop to read the user attributes on the line.
! 					 */
  			foreach(cur, attnumlist)
  			{
  				int			attnum = lfirst_int(cur);
  				int			m = attnum - 1;
+ 						char       *string;
  
! 						string = attr_buf.data + attr_offsets[m];
  
! 						if (nulls[m] == ' ')
! 							isnull = false;
  				else
! 							isnull = true;
  
  				if (csv_mode && isnull && force_notnull[m])
  				{
***************
*** 1806,1823 ****
  					copy_attname = NULL;
  				}
  			}
- 
- 			/*
- 			 * Complain if there are more fields on the input line.
- 			 *
- 			 * Special case: if we're reading a zero-column table, we won't
- 			 * yet have called CopyReadAttribute() at all; so no error if
- 			 * line is empty.
- 			 */
- 			if (result == NORMAL_ATTR && line_buf.len != 0)
- 				ereport(ERROR,
- 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- 					   errmsg("extra data after last expected column")));
  		}
  		else
  		{
--- 1784,1789 ----
***************
*** 1828,1834 ****
  			fld_count = CopyGetInt16();
  			if (CopyGetEof() || fld_count == -1)
  			{
! 				done = true;
  				break;
  			}
  
--- 1794,1801 ----
  			fld_count = CopyGetInt16();
  			if (CopyGetEof() || fld_count == -1)
  			{
! 						buf_done = true;
! 						no_more_data = true;
  				break;
  			}
  
***************
*** 1845,1851 ****
  					DatumGetObjectId(CopyReadBinaryAttribute(0,
  															 &oid_in_function,
  															 oid_typioparam,
- 															 -1,
  															 &isnull));
  				if (isnull || loaded_oid == InvalidOid)
  					ereport(ERROR,
--- 1812,1817 ----
***************
*** 1865,1871 ****
  				values[m] = CopyReadBinaryAttribute(i,
  													&in_functions[m],
  													typioparams[m],
- 													attr[m]->atttypmod,
  													&isnull);
  				nulls[m] = isnull ? 'n' : ' ';
  				copy_attname = NULL;
--- 1831,1836 ----
***************
*** 1885,1891 ****
  				nulls[defmap[i]] = ' ';
  		}
  
! 		/* Next apply any domain constraints */
  		if (hasConstraints)
  		{
  			ParamExecData *prmdata = &econtext->ecxt_param_exec_vals[0];
--- 1850,1858 ----
  				nulls[defmap[i]] = ' ';
  		}
  
! 				/*
! 				 * Next apply any domain constraints
! 				 */
  		if (hasConstraints)
  		{
  			ParamExecData *prmdata = &econtext->ecxt_param_exec_vals[0];
***************
*** 1912,1924 ****
  			}
  		}
  
! 		/* And now we can form the input tuple. */
  		tuple = heap_formtuple(tupDesc, values, nulls);
  
  		if (oids && file_has_oids)
  			HeapTupleSetOid(tuple, loaded_oid);
  
! 		/* Triggers and stuff need to be invoked in query context. */
  		MemoryContextSwitchTo(oldcontext);
  
  		skip_tuple = false;
--- 1879,1895 ----
  			}
  		}
  
! 				/*
! 				 * And now we can form the input tuple.
! 				 */
  		tuple = heap_formtuple(tupDesc, values, nulls);
  
  		if (oids && file_has_oids)
  			HeapTupleSetOid(tuple, loaded_oid);
  
! 				/*
! 				 * Triggers and stuff need to be invoked in query context.
! 				 */
  		MemoryContextSwitchTo(oldcontext);
  
  		skip_tuple = false;
***************
*** 1945,1955 ****
  			/* Place tuple in tuple slot */
  			ExecStoreTuple(tuple, slot, InvalidBuffer, false);
  
! 			/* Check the constraints of the tuple */
  			if (rel->rd_att->constr)
  				ExecConstraints(resultRelInfo, slot, estate);
  
! 			/* OK, store the tuple and create index entries for it */
  			simple_heap_insert(rel, tuple);
  
  			if (resultRelInfo->ri_NumIndices > 0)
--- 1916,1930 ----
  			/* Place tuple in tuple slot */
  			ExecStoreTuple(tuple, slot, InvalidBuffer, false);
  
! 					/*
! 					 * Check the constraints of the tuple
! 					 */
  			if (rel->rd_att->constr)
  				ExecConstraints(resultRelInfo, slot, estate);
  
! 					/*
! 					 * OK, store the tuple and create index entries for it
! 					 */
  			simple_heap_insert(rel, tuple);
  
  			if (resultRelInfo->ri_NumIndices > 0)
***************
*** 1958,1976 ****
  			/* AFTER ROW INSERT Triggers */
  			ExecARInsertTriggers(estate, resultRelInfo, tuple);
  		}
  	}
  
! 	/* Done, clean up */
  	error_context_stack = errcontext.previous;
  
  	MemoryContextSwitchTo(oldcontext);
  
! 	/* Execute AFTER STATEMENT insertion triggers */
  	ExecASInsertTriggers(estate, resultRelInfo);
  
! 	/* Handle queued AFTER triggers */
  	AfterTriggerEndQuery(estate);
  
  	pfree(values);
  	pfree(nulls);
  
--- 1933,1970 ----
  			/* AFTER ROW INSERT Triggers */
  			ExecARInsertTriggers(estate, resultRelInfo, tuple);
  		}
+ 
+ 				line_buf.len = 0;		/* we can reset line buffer now. */
+ 				line_buf.data[0] = '\0';
+ 				line_buf.cursor = 0;
+ 			}					/* end while(!buf_done) */
+ 		}						/* end if (bytesread > 0 || !fe_eof) */
+ 		else
+ 			/* no bytes read, end of data */
+ 		{
+ 			no_more_data = true;
  	}
+ 	} while (!no_more_data);
  
! 	/*
! 	 * Done, clean up
! 	 */
  	error_context_stack = errcontext.previous;
  
  	MemoryContextSwitchTo(oldcontext);
  
! 	/* 	 
!      * Execute AFTER STATEMENT insertion triggers
! 	 */
  	ExecASInsertTriggers(estate, resultRelInfo);
  
! 	/*
! 	 * Handle queued AFTER triggers
! 	 */
  	AfterTriggerEndQuery(estate);
  
+ 	pfree(attr_offsets);
+ 
  	pfree(values);
  	pfree(nulls);
  
***************
*** 1990,2262 ****
  
  
  /*
!  * Read the next input line and stash it in line_buf, with conversion to
!  * server encoding.
!  *
!  * Result is true if read was terminated by EOF, false if terminated
!  * by newline.
   */
  static bool
! CopyReadLine(char * quote, char * escape)
  {
! 	bool		result;
! 	bool		change_encoding = (client_encoding != server_encoding);
! 	int			c;
! 	int			mblen;
! 	int			j;
! 	unsigned char s[2];
  	char	   *cvt;
! 	bool        in_quote = false, last_was_esc = false, csv_mode = false;
! 	char        quotec = '\0', escapec = '\0';
! 
! 	if (quote)
! 	{
! 		csv_mode = true;
! 		quotec = quote[0];
! 		escapec = escape[0];
! 		/* ignore special escape processing if it's the same as quotec */
! 		if (quotec == escapec)
! 			escapec = '\0';
! 	}
! 
! 
! 	s[1] = 0;
! 
! 	/* reset line_buf to empty */
! 	line_buf.len = 0;
! 	line_buf.data[0] = '\0';
! 	line_buf.cursor = 0;
  
  	/* mark that encoding conversion hasn't occurred yet */
  	line_buf_converted = false;
  
! 	/* set default status */
! 	result = false;
  
  	/*
! 	 * In this loop we only care for detecting newlines (\r and/or \n) and
! 	 * the end-of-copy marker (\.).  
! 	 *
! 	 * In Text mode, for backwards compatibility we allow
! 	 * backslashes to escape newline characters.  Backslashes other than
! 	 * the end marker get put into the line_buf, since CopyReadAttribute
! 	 * does its own escape processing.	
! 	 *
! 	 * In CSV mode, CR and NL inside q quoted field are just part of the
! 	 * data value and are put in line_buf. We keep just enough state
! 	 * to know if we are currently in a quoted field or not.
! 	 *
! 	 * These four characters, and only these four, are assumed the same in 
! 	 * frontend and backend encodings.
! 	 *
! 	 * We do not assume that second and later bytes of a frontend
! 	 * multibyte character couldn't look like ASCII characters.
  	 */
! 	for (;;)
  	{
! 		c = CopyGetChar();
! 		if (c == EOF)
  		{
! 			result = true;
! 			break;
  		}
  
! 		if (csv_mode)
  		{
  			/*  
! 			 * Dealing with quotes and escapes here is mildly tricky. If the
! 			 * quote char is also the escape char, there's no problem - we  
! 			 * just use the char as a toggle. If they are different, we need
! 			 * to ensure that we only take account of an escape inside a quoted
! 			 * field and immediately preceding a quote char, and not the
! 			 * second in a escape-escape sequence.
  			 */ 
  
! 			if (in_quote && c == escapec)
! 				last_was_esc = ! last_was_esc;
! 			if (c == quotec && ! last_was_esc)
! 				in_quote = ! in_quote;
! 			if (c != escapec)
! 				last_was_esc = false;
  
  			/*
! 			 * updating the line count for embedded CR and/or LF chars is 
! 			 * necessarily a little fragile - this test is probably about 
! 			 * the best we can do.
  			 */ 
! 			if (in_quote && c == (eol_type == EOL_CR ? '\r' : '\n')) 
! 				copy_lineno++; 
! 		}
  
! 		if (!in_quote && c == '\r')
  		{
! 			if (eol_type == EOL_NL)
  			{
! 				if (! csv_mode)
! 					ereport(ERROR,
! 							(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 							 errmsg("literal carriage return found in data"),
! 							 errhint("Use \"\\r\" to represent carriage return.")));
! 				else
! 					ereport(ERROR,
! 							(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 							 errmsg("unquoted carriage return found in CSV data"),
! 							 errhint("Use quoted CSV field to represent carriage return.")));
  			}
! 			/* Check for \r\n on first line, _and_ handle \r\n. */
! 			if (eol_type == EOL_UNKNOWN || eol_type == EOL_CRNL)
  			{
! 				int			c2 = CopyPeekChar();
  
! 				if (c2 == '\n')
! 				{
! 					CopyDonePeek(c2, true);		/* eat newline */
! 					eol_type = EOL_CRNL;
  				}
  				else
  				{
! 					/* found \r, but no \n */
  					if (eol_type == EOL_CRNL)
  					{
! 						if (!csv_mode)
! 							ereport(ERROR,
! 									(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 									 errmsg("literal carriage return found in data"),
! 									 errhint("Use \"\\r\" to represent carriage return.")));
  						else
! 							ereport(ERROR,
! 									(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 									 errmsg("unquoted carriage return found in data"),
! 									 errhint("Use quoted CSV field to represent carriage return.")));
! 
  					}
  
  					/*
! 					 * if we got here, it is the first line and we didn't
! 					 * get \n, so put it back
  					 */
! 					CopyDonePeek(c2, false);
! 					eol_type = EOL_CR;
  				}
  			}
- 			break;
  		}
! 		if (!in_quote && c == '\n')
  		{
! 			if (eol_type == EOL_CR || eol_type == EOL_CRNL)
! 			{
! 				if (!csv_mode)
! 					ereport(ERROR,
! 							(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 							 errmsg("literal newline found in data"),
! 							 errhint("Use \"\\n\" to represent newline.")));
! 				else
! 					ereport(ERROR,
! 							(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 							 errmsg("unquoted newline found in data"),
! 							 errhint("Use quoted CSV field to represent newline.")));
! 					
  			}
- 			eol_type = EOL_NL;
- 			break;
- 		}
  
! 		if ((line_buf.len == 0 || !csv_mode) && c == '\\')
! 		{
! 			int c2;
! 			
! 			if (csv_mode)
! 				c2 = CopyPeekChar();
! 			else
! 				c2 = c = CopyGetChar();
  
! 			if (c2 == EOF)
  			{
! 				result = true;
! 				if (csv_mode)
! 					CopyDonePeek(c2, true);
  				break;
  			}
! 			if (c2 == '.')
! 			{
! 				if (csv_mode)
! 					CopyDonePeek(c2, true); /* allow keep calling GetChar() */
  
! 				if (eol_type == EOL_CRNL)
! 				{
! 					c = CopyGetChar();
! 					if (c == '\n')
! 						ereport(ERROR,
! 								(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 								 errmsg("end-of-copy marker does not match previous newline style")));
! 					if (c != '\r')
! 						ereport(ERROR,
! 								(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 								 errmsg("end-of-copy marker corrupt")));
  				}
- 				c = CopyGetChar();
- 				if (c != '\r' && c != '\n')
- 					ereport(ERROR,
- 							(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- 							 errmsg("end-of-copy marker corrupt")));
- 				if ((eol_type == EOL_NL && c != '\n') ||
- 					(eol_type == EOL_CRNL && c != '\n') ||
- 					(eol_type == EOL_CR && c != '\r'))
- 					ereport(ERROR,
- 							(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- 							 errmsg("end-of-copy marker does not match previous newline style")));
  
  				/*
! 				 * In protocol version 3, we should ignore anything after
! 				 * \. up to the protocol end of copy data.	(XXX maybe
! 				 * better not to treat \. as special?)
  				 */
! 				if (copy_dest == COPY_NEW_FE)
  				{
! 					while (c != EOF)
! 						c = CopyGetChar();
  				}
! 				result = true;	/* report EOF */
  				break;
  			}
  			
! 			if (csv_mode)
! 				CopyDonePeek(c2, false); /* not a dot, so put it back */ 
  			else
! 				/* not EOF mark, so emit \ and following char literally */
! 				appendStringInfoCharMacro(&line_buf, '\\');
  		}
  
! 		appendStringInfoCharMacro(&line_buf, c);
  
  		/*
! 		 * When client encoding != server, must be careful to read the
! 		 * extra bytes of a multibyte character exactly, since the
! 		 * encoding might not ensure they don't look like ASCII.  When the
! 		 * encodings are the same, we need not do this, since no server
! 		 * encoding we use has ASCII-like following bytes.
  		 */
! 		if (change_encoding)
! 		{
! 			s[0] = c;
! 			mblen = pg_encoding_mblen(client_encoding, s);
! 			for (j = 1; j < mblen; j++)
! 			{
! 				c = CopyGetChar();
! 				if (c == EOF)
  				{
! 					result = true;
  					break;
  				}
- 				appendStringInfoCharMacro(&line_buf, c);
  			}
! 			if (result)
! 				break;			/* out of outer loop */
  		}
- 	}							/* end of outer loop */
  
! 	/* Done reading the line.  Convert it to server encoding. */
! 	if (change_encoding)
  	{
  		cvt = (char *) pg_client_to_server((unsigned char *) line_buf.data,
  										   line_buf.len);
--- 1984,2422 ----
  
  
  /*
!  * Finds the next TEXT line that is in the input buffer and loads 
!  * it into line_buf. Returns an indication if the line that was read 
!  * is complete (if an unescaped line-end was encountered). If we 
!  * reached the end of buffer before the whole line was written into the
!  * line buffer then returns false.
   */
  static bool
! CopyReadLineText(size_t bytesread, char *escape)
  {
! 	int			linesize;
! 	bool		transcode = (client_encoding != server_encoding);
  	char	   *cvt;
! 	char        escapec = '\0';
  
  	/* mark that encoding conversion hasn't occurred yet */
  	line_buf_converted = false;
  
! 	/*
! 	 * set the escape char for text format ('\\' by default).
! 	 */
! 	escapec = escape[0];
  
  	/*
! 	 * Detect end of line type if not already detected.
  	 */
! 	if (eol_type == EOL_UNKNOWN)
  	{
! 		bool		eol_detected = DetectLineEnd(bytesread, NULL, escape);
! 
! 		if (!eol_detected)
  		{
! 			/* load entire input buffer into line buf, and quit */
! 			appendBinaryStringInfo(&line_buf, input_buf, COPY_BUF_SIZE);
! 			buf_done = true;
! 			return false;
! 		}
  		}
  
! 	/*
! 	 * Special case: eol is CRNL, last byte of previous buffer was an
! 	 * unescaped CR and 1st byte of current buffer is NL. We check for
! 	 * that here.
! 	 */
! 	if (eol_type == EOL_CRNL)
! 	{
! 		/* if we started scanning from the 1st byte of the buffer */
! 		if (begloc == input_buf)
! 		{
! 			/* and had a CR in last byte of prev buf */
! 			if (cr_in_prevbuf)
! 			{
! 				/*
! 				 * if this 1st byte in buffer is 2nd byte of line end sequence
! 				 * (linefeed)
! 				 */
! 				if (*begloc == eol_ch[1])
  		{
  			/*  
! 					 * load that one linefeed byte and indicate we are done
! 					 * with the data line
  			 */ 
+ 					appendBinaryStringInfo(&line_buf, begloc, 1);
+ 					buffer_index++;
+ 					begloc++;
+ 					esc_in_prevbuf = false;
+ 					cr_in_prevbuf = false;
+ 					return true;
+ 				}
+ 			}
  
! 			cr_in_prevbuf = false;
! 		}
! 	}
  
  			/*
! 	 * (we need a loop so that if eol_ch is found, but prev ch is backslash,
! 	 * we can search for the next eol_ch)
  			 */ 
! 	while (true)
! 	{
! 		/* reached end of buffer */
! 		if ((endloc = scanTextLine(begloc, eol_ch[0], bytesread - buffer_index)) == NULL)
! 		{
! 			linesize = COPY_BUF_SIZE - (begloc - input_buf);
! 			appendBinaryStringInfo(&line_buf, begloc, linesize);
  
! 			if (line_buf.len > 1)
  		{
! 				char	   *last_ch = line_buf.data + line_buf.len - 1; /* before terminating \0 */
! 
! 				if (*last_ch == escapec)
  			{
! 					esc_in_prevbuf = true;
! 
! 					if (line_buf.len > 2)
! 					{
! 						last_ch--;
! 						if (*last_ch == escapec)
! 							esc_in_prevbuf = false;
  			}
! 				}
! 				else if (*last_ch == '\r')
  			{
! 					if (eol_type == EOL_CRNL)
! 						cr_in_prevbuf = true;
! 				}
! 			}
  
! 			line_done = false;
! 			buf_done = true;
! 			break;
  				}
  				else
+ 			/* found the 1st eol ch in input_buf. */
  				{
! 			bool		eol_found = true;
! 			bool		eol_escaped = true;
! 
! 			/*
! 			 * Load that piece of data (potentially a data line) into the line buffer,
! 			 * and update the pointers for the next scan.
! 			 */
! 			linesize = endloc - begloc + 1;
! 			appendBinaryStringInfo(&line_buf, begloc, linesize);
! 			buffer_index += linesize;
! 			begloc = endloc + 1;
! 
  					if (eol_type == EOL_CRNL)
  					{
! 				/* check if there is a '\n' after the '\r' */
! 				if (*(endloc + 1) == '\n')
! 				{
! 					/* this is a line end */
! 					appendBinaryStringInfo(&line_buf, begloc, 1);		/* load that '\n' */
! 					buffer_index++;
! 					begloc++;
! 				}
  						else
! 					/* just a CR, not a line end */
! 					eol_found = false;
  					}
  
  					/*
! 			 * in some cases, this end of line char happens to be the
! 			 * last character in the buffer. we need to catch that.
  					 */
! 			if (buffer_index >= bytesread)
! 				buf_done = true;
! 
! 			/*
! 			 * Check if the 1st end of line ch is escaped.
! 			 */
! 			if (endloc != input_buf)	/* can we look 1 char back? */
! 			{
! 				if (*(endloc - 1) != escapec)	/* prev char is not an escape */
! 					eol_escaped = false;
! 				else
! 					/* prev char is an escape */
! 				{
! 					if (endloc != (input_buf + 1))		/* can we look another
! 														 * char back? */
! 					{
! 						/* it's a double escape char, so it's not an escape */
! 						if (*(endloc - 2) == escapec)
! 							eol_escaped = false;
! 						/* else it's a single escape char, so EOL is ascaped */
! 					}
! 					else
! 					{
! 						/* we need to check in the last buffer */
! 						if (esc_in_prevbuf)		
! 							eol_escaped = false;
  				}
  			}
  		}
! 			else
! 				/*
! 				 * this eol ch is first ch in buffer, check for escape in prev buf
! 				 */
  		{
! 				if (!esc_in_prevbuf)
! 					eol_escaped = false;
  			}
  
! 			esc_in_prevbuf = false;		/* reset variable */
  
! 			/*
! 			 * if eol was found, and it isn't escaped, line is done
! 			 */
! 			if ((eol_escaped == false) && eol_found)
  			{
! 				line_done = true;
  				break;
  			}
! 				else
! 				/* stay in the loop and process some more data. */
! 				line_done = false;
! 					
! 		}						/* end of found eol_ch */
! 		}
  
! 	/*
! 	 * Done reading the line. Convert it to server encoding.
! 	 */
! 	if (transcode)
! 	{
! 		cvt = (char *) pg_client_to_server((unsigned char *) line_buf.data,
! 										   line_buf.len);
! 		if (cvt != line_buf.data)
! 		{
! 			/* transfer converted data back to line_buf */
! 			line_buf.len = 0;
! 			line_buf.data[0] = '\0';
! 			line_buf.cursor = 0;
! 			appendBinaryStringInfo(&line_buf, cvt, strlen(cvt));
! 		}
! 	}
! 			
! 	/* indicate that conversion had occured */
! 	line_buf_converted = true;
! 
! 	/*
! 	 * check if this line is an end marker -- "\."
! 	 */
! 	end_marker = false;
! 
! 	switch (eol_type)
! 			{
! 		case EOL_NL:
! 			if (!strcmp(line_buf.data, "\\.\n"))
! 				end_marker = true;
! 			break;
! 		case EOL_CR:
! 			if (!strcmp(line_buf.data, "\\.\r"))
! 				end_marker = true;
! 			break;
! 		case EOL_CRNL:
! 			if (!strcmp(line_buf.data, "\\.\r\n"))
! 				end_marker = true;
! 			break;
! 		case EOL_UNKNOWN:
! 				break;
! 			}
! 
! 	if (end_marker)
! 			{
! 		fe_eof = true;
! 		/* we don't want to process a \. as data line, want to quit. */
! 		line_done = false;
! 		buf_done = true;
! 	}
! 
! 	return line_done;
! }
! 
! /*
!  * Finds the next CSV line that is in the input buffer and loads 
!  * it into line_buf. Returns an indication if the line that was read 
!  * is complete (if an unescaped line-end was encountered). If we 
!  * reached the end of buffer before the whole line was written into the
!  * line buffer then returns false.
!  */
! static bool
! CopyReadLineCSV(size_t bytesread, char *quote, char *escape)
! {
! 	int			linesize;
! 	char	   *cvt;
! 	char        quotec = '\0',
!                 escapec = '\0';
! 	bool		transcode = (client_encoding != server_encoding);
! 	
! 	/* mark that encoding conversion hasn't occurred yet */
! 	line_buf_converted = false;
! 	
! 	escapec = escape[0];
! 	quotec = quote[0];
! 	
!     /* ignore special escape processing if it's the same as quotec */
! 	if (quotec == escapec)
! 		escapec = '\0';
! 
! 	/*
! 	 * Detect end of line type if not already detected.
! 	 */
! 	if (eol_type == EOL_UNKNOWN)
! 	{
! 		bool		eol_detected = DetectLineEnd(bytesread, quote, escape);
! 		
! 		if (!eol_detected)
! 		{
! 			/* load entire input buffer into line buf, and quit */
! 			appendBinaryStringInfo(&line_buf, input_buf, COPY_BUF_SIZE);
! 			line_done = false;
! 			buf_done = true;
! 			
! 			return line_done;
! 		}
! 	}
! 	
! 	/*
! 	 * Special case: eol is CRNL, last byte of previous buffer was an
! 	 * unescaped CR and 1st byte of current buffer is NL. We check for
! 	 * that here.
! 	 */
! 				if (eol_type == EOL_CRNL)
! 				{
! 		/* if we started scanning from the 1st byte of the buffer */
! 		if (begloc == input_buf)
! 		{
! 			/* and had a CR in last byte of prev buf */
! 			if (cr_in_prevbuf)
! 			{
! 				/*
! 				 * if this 1st byte in buffer is 2nd byte of line end sequence
! 				 * (linefeed)
! 				 */
! 				if (*begloc == eol_ch[1])
! 				{
! 					/*
! 					 * load that one linefeed byte and indicate we are done
! 					 * with the data line
! 					 */
! 					appendBinaryStringInfo(&line_buf, begloc, 1);
! 					buffer_index++;
! 					begloc++;
! 					
! 					line_done = true;
! 					esc_in_prevbuf = false;
! 					cr_in_prevbuf = false;
! 					
! 					return line_done;
! 				}
! 			}
! 			
! 			cr_in_prevbuf = false;
! 		}
  				}
  
  				/*
! 	 * (we need a loop so that if eol_ch is found, but we are in quotes,
! 	 * we can search for the next eol_ch)
  				 */
! 	while (true)
! 	{
! 		/* reached end of buffer */
! 		if ((endloc = scanCSVLine(begloc, eol_ch[0], escapec, quotec, bytesread - buffer_index)) == NULL)
  				{
! 			linesize = COPY_BUF_SIZE - (begloc - input_buf);
! 			appendBinaryStringInfo(&line_buf, begloc, linesize);
! 			
! 			if (last_was_esc)
! 				esc_in_prevbuf = true;
! 			
! 			if (line_buf.len > 1)
! 			{
! 				char	   *last_ch = line_buf.data + line_buf.len - 1; /* before terminating \0 */
! 				
! 				if (*last_ch == '\r')
! 				{
! 					if (eol_type == EOL_CRNL)
! 						cr_in_prevbuf = true;
  				}
! 			}
! 			
! 			line_done = false;
! 			buf_done = true;
  				break;
  			}
+ 		else
+ 			/* found 1st eol char in input_buf. */
+ 		{
+ 			bool		eol_found = true;
  			
! 			/*
! 			 * Load that piece of data (potentially a data line) into the line buffer,
! 			 * and update the pointers for the next scan.
! 			 */
! 			linesize = endloc - begloc + 1;
! 			appendBinaryStringInfo(&line_buf, begloc, linesize);
! 			buffer_index += linesize;
! 			begloc = endloc + 1;
! 			
! 			/* end of line only if not in quotes */
! 			if(in_quote)
! 			{
! 				line_done = false;
! 				/* buf done, but still in quote */
! 				if (buffer_index >= bytesread)
! 				{
! 					buf_done = true;
! 					break;
! 				}
! 			}
  			else
! 			{
! 				/* if dos eol, check for '\n' after the '\r' */
! 				if (eol_type == EOL_CRNL)
! 				{
! 					if (*(endloc + 1) == '\n')
! 					{
! 						/* this is a line end */
! 						appendBinaryStringInfo(&line_buf, begloc, 1);		/* load that '\n' */
! 						buffer_index++;
! 						begloc++;
! 					}
! 			else
! 						/* just a CR, not a line end */
! 						eol_found = false;
  		}
  
! 				/*
! 				 * in some cases, this end of line char happens to be the
! 				 * last character in the buffer. we need to catch that.
! 				 */
! 				if (buffer_index >= bytesread)
! 					buf_done = true;
  
  		/*
! 				 * if eol was found line is done
  		 */
! 				if (eol_found)
  				{
! 					line_done = true;
  					break;
  				}
  			}
! 		}						/* end of found eol_ch */
  		}
  
! 	/*
! 	 * Done reading the line. Convert it to server encoding.
! 	 */
! 	if (transcode)
  	{
  		cvt = (char *) pg_client_to_server((unsigned char *) line_buf.data,
  										   line_buf.len);
***************
*** 2265,2285 ****
  			/* transfer converted data back to line_buf */
  			line_buf.len = 0;
  			line_buf.data[0] = '\0';
  			appendBinaryStringInfo(&line_buf, cvt, strlen(cvt));
  		}
  	}
  
! 	/* Now it's safe to use the buffer in error messages */
  	line_buf_converted = true;
  
! 	return result;
  }
  
  /*
   *	Return decimal value for a hexadecimal digit
   */
  static
! int GetDecimalFromHex(char hex)
  {
  	if (isdigit(hex))
  		return hex - '0';
--- 2425,2478 ----
  			/* transfer converted data back to line_buf */
  			line_buf.len = 0;
  			line_buf.data[0] = '\0';
+ 			line_buf.cursor = 0;
  			appendBinaryStringInfo(&line_buf, cvt, strlen(cvt));
  		}
  	}
  
! 	/* indicate that conversion had occured */
  	line_buf_converted = true;
  
! 	/*
! 	 * check if this line is an end marker -- "\."
! 	 */
! 	end_marker = false;
! 
! 	switch (eol_type)
! 	{
! 		case EOL_NL:
! 			if (!strcmp(line_buf.data, "\\.\n"))
! 				end_marker = true;
! 			break;
! 		case EOL_CR:
! 			if (!strcmp(line_buf.data, "\\.\r"))
! 				end_marker = true;
! 			break;
! 		case EOL_CRNL:
! 			if (!strcmp(line_buf.data, "\\.\r\n"))
! 				end_marker = true;
! 			break;
! 		case EOL_UNKNOWN:
! 			break;
! 	}
! 
! 	if (end_marker)
! 	{
! 		fe_eof = true;
! 		/* we don't want to process a \. as data line, want to quit. */
! 		line_done = false;
! 		buf_done = true;
! 	}
! 
! 	return line_done;
  }
  
  /*
   *	Return decimal value for a hexadecimal digit
   */
  static
! int
! GetDecimalFromHex(char hex)
  {
  	if (isdigit(hex))
  		return hex - '0';
***************
*** 2287,2344 ****
  		return tolower(hex) - 'a' + 10;
  }
  
! /*----------
!  * Read the value of a single attribute, performing de-escaping as needed.
!  *
!  * delim is the column delimiter string (must be just one byte for now).
!  * null_print is the null marker string.  Note that this is compared to
!  * the pre-de-escaped input string.
!  *
!  * *result is set to indicate what terminated the read:
!  *		NORMAL_ATTR:	column delimiter
!  *		END_OF_LINE:	end of line
!  * In either case, the string read up to the terminator is returned.
!  *
!  * *isnull is set true or false depending on whether the input matched
!  * the null marker.  Note that the caller cannot check this since the
!  * returned string will be the post-de-escaping equivalent, which may
!  * look the same as some valid data string.
!  *----------
   */
! static char *
! CopyReadAttribute(const char *delim, const char *null_print,
! 				  CopyReadResult *result, bool *isnull)
  {
  	char		c;
! 	char		delimc = delim[0];
! 	int			start_cursor = line_buf.cursor;
! 	int			end_cursor;
! 	int			input_len;
  
! 	/* reset attribute_buf to empty */
! 	attribute_buf.len = 0;
! 	attribute_buf.data[0] = '\0';
  
! 	/* set default status */
! 	*result = END_OF_LINE;
  
! 	for (;;)
  	{
! 		end_cursor = line_buf.cursor;
! 		if (line_buf.cursor >= line_buf.len)
! 			break;
! 		c = line_buf.data[line_buf.cursor++];
! 		if (c == delimc)
  		{
! 			*result = NORMAL_ATTR;
! 			break;
  		}
! 		if (c == '\\')
  		{
! 			if (line_buf.cursor >= line_buf.len)
! 				break;
! 			c = line_buf.data[line_buf.cursor++];
! 			switch (c)
  			{
  				case '0':
  				case '1':
--- 2480,2769 ----
  		return tolower(hex) - 'a' + 10;
  }
  
! /*
!  * Detected the eol type by looking at the first data row.
!  * Possible eol types are NL, CR, or CRNL. If eol type was
!  * detected, it is set and a boolean true is returned to
!  * indicated detection was successful. If the first data row
!  * is longer than the input buffer, we return false and will
!  * try again in the next buffer.
   */
! static bool
! DetectLineEnd(size_t bytesread, char *quote, char *escape)
  {
+ 	int         index = 0;
  	char		c;
! 	char		quotec = '\0',
! 		escapec = '\0';
! 	bool        csv = false;
  
! 	if (quote) /* CSV format */
! 	{
! 		csv = true;
! 		quotec = quote[0];
! 		escapec = escape[0];
! 		/* ignore special escape processing if it's the same as quotec */
! 		if (quotec == escapec)
! 			escapec = '\0';
! 	}
  
! 	while (index < COPY_BUF_SIZE)
! 	{
! 		c = input_buf[index];
  
! 		if(csv)
  	{
! 			if (in_quote && c == escapec)
! 				last_was_esc = !last_was_esc;
! 			if (c == quotec && !last_was_esc)
! 				in_quote = !in_quote;
! 			if (c != escapec)
! 				last_was_esc = false;
! 		}
! 		
! 		if (c == '\n')
  		{
! 			if(!csv || (csv && !in_quote))
! 			{
! 				eol_type = EOL_NL;
! 				eol_ch[0] = '\n';
! 				eol_ch[1] = '\0';
! 				
! 				in_quote = false;
! 				last_was_esc = false;
! 				return true;
! 			}
  		}
! 		if (c == '\r')
  		{
! 			if(!csv || (csv && !in_quote))
!             {
! 				if (input_buf[index + 1] == '\n') /* always safe */
! 					{
! 					eol_type = EOL_CRNL;
! 					eol_ch[0] = '\r';
! 					eol_ch[1] = '\n';
! 					}
! 					else
! 					{
! 						eol_type = EOL_CR;
! 						eol_ch[0] = '\r';
! 						eol_ch[1] = '\0';
! 					}
! 					
! 					in_quote = false;
! 					last_was_esc = false;
! 					return true;
! 			}
! 		}
! 
! 		index++;
! 	}
! 
! return false;
! }
! 
! /*
!  * Read all TEXT attributes. Attributes are parsed from line_buf and
!  * inserted (all at once) to attr_buf, while saving pointers to
!  * each attribute's starting position.
!  *
!  * When this routine finishes execution both the nulls array and
!  * the attr_offsets array are updated. The attr_offsets will include
!  * the offset from the beginning of the attribute array of which
!  * each attribute begins. If a specific attribute is not used for this
!  * COPY command (ommitted from the column list), a value of 0 will be assigned.
!  * For example: for table foo(a,b,c,d,e) and COPY foo(a,b,e)
!  * attr_offsets may look something like this after this routine
!  * returns: [0,20,0,0,55]. That means that column "a" value starts
!  * at byte offset 0, "b" in 20 and "e" in 55, in attr_buf.
!  *
!  * In the attribute buffer (attr_buf) each attribute
!  * is terminated with a '\0', and therefore by using the attr_offsets
!  * array we could point to a beginning of an attribute and have it
!  * behave as a C string, much like previously done in COPY.
!  *
!  * Another aspect to improving performance is reducing the frequency
!  * of data load into buffers. The original COPY read attribute code
!  * loaded a character at a time. In here we try to load a chunk of data
!  * at a time. Usually a chunk will include a full data row
!  * (unless we have an escaped delim). That effectively reduces the number of
!  * loads by a factor of number of bytes per row. This improves performance
!  * greatly, unfortunately it add more complexity to the code.
!  *
!  * Global participants in parsing logic:
!  *
!  * line_buf.cursor -- an offset from beginning of the line buffer
!  * that indicates where we are about to begin the next scan. Note that
!  * if we have WITH OIDS this cursor is already shifted past the first
!  * OID attribute.
!  *
!  * attr_buf.cursor -- an offset from the beginning of the
!  * attribute buffer that indicates where the current attribute begins.
!  */
! static void 
! CopyReadAttributesText(const char *delim, const char *escape, const char *null_print,
! 					   int null_print_len, char *nulls, List *attnumlist, 
! 					   int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr)
! {
! 	char		delimc = delim[0];		/* delimiter character */
! 	char        escapec = escape[0];    /* escape character    */
! 	char	   *scan_start;		/* pointer to line buffer for scan start. */
! 	char	   *scan_end;		/* pointer to line buffer where char was found */
! 	int			attr_pre_len;	/* attr raw len, before processing escapes */
! 	int			attr_post_len;	/* current attr len after escaping */
! 	int			m;				/* attribute index being parsed */
! 	int			bytes_remaining;/* num bytes remaining to be scanned in line buf */
! 	int			chunk_start;	/* offset to beginning of line chunk to load */
! 	int			chunk_len;		/* length of chunk of data to load to attr buf */
! 	int			oct_val;		/* byte value for octal escapes */
! 	int         hex_val;        /* byte value for hexadecimal escapes */
! 	char        hexchar;        /* char that appears after \x for hex values */
! 	int			attnum;			/* attribute number being parsed */
! 	ListCell   *cur;			/* cursor to attribute list used for this COPY */
! 	int			attribute;
! 		
! 		/*
! 		 * init variables for attribute scan
! 		 */
! 		attr_buf.len = 0;
! 		attr_buf.data[0] = '\0';
! 		attr_buf.cursor = 0;
! 		/* cursor is now > 0 if we copy WITH OIDS */
! 		scan_start = line_buf.data + line_buf.cursor;
! 		cur = list_head(attnumlist);
! 		attnum = lfirst_int(cur);
! 		m = attnum - 1;
! 		chunk_start = line_buf.cursor;
! 		chunk_len = 0;
! 		attr_pre_len = 0;
! 		attr_post_len = 0;
! 		
! 		/*
! 		 * Scan through the line buffer to read all attributes data
! 		 */
! 		while (line_buf.cursor < line_buf.len)
! 		{
! 			bytes_remaining = line_buf.len - line_buf.cursor;
! 			
! 			if ((scan_end = scanTextAttr(scan_start, delimc, escapec, bytes_remaining))
! 				== NULL)
! 			{
! 				/* GOT TO END OF LINE BUFFER */
! 				
! 				if (cur == NULL)
! 					ereport(ERROR,
! 							(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 							 errmsg("extra data after last expected column")));
! 				
! 				attnum = lfirst_int(cur);
! 				m = attnum - 1;
! 				
! 				/* don't count eol char(s) in attr and chunk len calculation */
! 				if (eol_type == EOL_CRNL)
! 				{
! 					attr_pre_len += bytes_remaining - 2;
! 					chunk_len = line_buf.len - chunk_start - 2;
! 				}
! 				else
! 				{
! 					attr_pre_len += bytes_remaining - 1;
! 					chunk_len = line_buf.len - chunk_start - 1;
! 				}
! 				
! 				/* check if this is a NULL value or data value (assumed NULL) */
! 				if (attr_pre_len == null_print_len
! 					&&
! 					strncmp(line_buf.data + line_buf.len - attr_pre_len - 1, null_print, attr_pre_len)
! 					== 0)
! 					nulls[m] = 'n';
! 				else
! 					nulls[m] = ' ';
! 				
! 				attr_offsets[m] = attr_buf.cursor;
! 				
! 				
! 				/* load the last chunk, the whole buffer in most cases */
! 				appendBinaryStringInfo(&attr_buf, line_buf.data + chunk_start, chunk_len);
! 				
! 				line_buf.cursor += attr_pre_len + 2;		/* skip eol char and
! 					* '\0' to exit loop */
! 				
! 				if (lnext(cur) != NULL)
! 				{
! 					/*
! 					 * For an empty data line, the previous COPY code will
! 					 * fail it during the conversion stage. We can fail it here
! 					 * already, but then we will fail the regression tests b/c
! 					 * of a different error message. that's why we return so we
! 					 * can get the same error message that regress expects. ahh...
! 					 * this conditional is unnecessary and should be removed soon.
! 					 */
! 					if (line_buf.len > 1)
! 						ereport(ERROR,
! 								(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 								 errmsg("missing data for column \"%s\"",
! 										NameStr(attr[m + 1]->attname))));
! 					else
! 						return;
! 				}
! 			}
! 			else
! 				/* FOUND A DELIMITER OR ESCAPE */
! 			{
! 				if (cur == NULL)
! 					ereport(ERROR,
! 							(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 							 errmsg("extra data after last expected column")));
! 				
! 				if (*scan_end == delimc)		/* found a delimiter */
! 				{
! 					attnum = lfirst_int(cur);
! 					m = attnum - 1;
! 					
! 					/* (we don't include the delimiter ch in length) */
! 					attr_pre_len += scan_end - scan_start;
! 					/* (we don't include the delimiter ch in length) */
! 					attr_post_len += scan_end - scan_start;
! 					
! 					/* check if this is a null print or data (assumed NULL) */
! 					if (attr_pre_len == null_print_len
! 						&&
! 						strncmp(scan_end - attr_pre_len, null_print, attr_pre_len)
! 						== 0)
! 						nulls[m] = 'n';
! 					else
! 						nulls[m] = ' ';
! 					
! 					/* set the pointer to next attribute position */
! 					attr_offsets[m] = attr_buf.cursor;
! 					
! 					/*
! 					 * update buffer cursors to our current location, +1 to skip
! 					 * the delimc
! 					 */
! 					line_buf.cursor = scan_end - line_buf.data + 1;
! 					attr_buf.cursor += attr_post_len + 1;
! 					
! 					/* prepare scan for next attr */
! 					scan_start = line_buf.data + line_buf.cursor;
! 					cur = lnext(cur);
! 					attr_pre_len = 0;
! 					attr_post_len = 0;
! 				}
! 				else
! 					/* found an escape character */
! 				{
! 					char		nextc = *(scan_end + 1);
! 					char		newc;
! 					int			skip = 2;
! 					
! 					chunk_len = (scan_end - line_buf.data) - chunk_start + 1;
! 					
! 					/* load a chunk of data */
! 					appendBinaryStringInfo(&attr_buf, line_buf.data + chunk_start, chunk_len);
! 					
! 					switch (nextc)
  			{
  				case '0':
  				case '1':
***************
*** 2348,2445 ****
  				case '5':
  				case '6':
  				case '7':
! 					/* handle \013 */
! 					{
! 						int			val;
  
! 						val = OCTVALUE(c);
! 						if (line_buf.cursor < line_buf.len)
! 						{
! 							c = line_buf.data[line_buf.cursor];
! 							if (ISOCTAL(c))
! 							{
! 								line_buf.cursor++;
! 								val = (val << 3) + OCTVALUE(c);
! 								if (line_buf.cursor < line_buf.len)
  								{
! 									c = line_buf.data[line_buf.cursor];
! 									if (ISOCTAL(c))
  									{
! 										line_buf.cursor++;
! 										val = (val << 3) + OCTVALUE(c);
  									}
  								}
! 							}
! 						}
! 						c = val & 0377;
! 					}
  					break;
  				case 'x':
  					/* Handle \x3F */
! 					if (line_buf.cursor < line_buf.len)
! 					{
! 						char hexchar = line_buf.data[line_buf.cursor];
  
  						if (isxdigit(hexchar))
  						{
! 							int val = GetDecimalFromHex(hexchar);
  
! 							line_buf.cursor++;
! 							if (line_buf.cursor < line_buf.len)
! 							{
! 								hexchar = line_buf.data[line_buf.cursor];
  								if (isxdigit(hexchar))
  								{
! 									line_buf.cursor++;
! 									val = (val << 4) + GetDecimalFromHex(hexchar);
! 								}
! 							}
! 							c = val & 0xff;
  						}
  					}
  					break;
  				case 'b':
! 					c = '\b';
  					break;
  				case 'f':
! 					c = '\f';
  					break;
  				case 'n':
! 					c = '\n';
  					break;
  				case 'r':
! 					c = '\r';
  					break;
  				case 't':
! 					c = '\t';
  					break;
  				case 'v':
! 					c = '\v';
  					break;
  
  					/*
! 					 * in all other cases, take the char after '\'
! 					 * literally
  					 */
  			}
! 		}
! 		appendStringInfoCharMacro(&attribute_buf, c);
  	}
  
! 	/* check whether raw input matched null marker */
! 	input_len = end_cursor - start_cursor;
! 	if (input_len == strlen(null_print) &&
! 		strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0)
  		*isnull = true;
! 	else
! 		*isnull = false;
  
! 	return attribute_buf.data;
! }
  
  
  /*
!  * Read the value of a single attribute in CSV mode,
   * performing de-escaping as needed. Escaping does not follow the normal
   * PostgreSQL text mode, but instead "standard" (i.e. common) CSV usage.
   *
--- 2773,2944 ----
  				case '5':
  				case '6':
  				case '7':
! 							oct_val = OCTVALUE(nextc);
! 							nextc = *(scan_end + 2);
  
! 							/*
! 							 * (no need for out bad access check since line if
! 								* buffered)
! 							 */
! 							if (ISOCTAL(nextc))
  								{
! 								skip++;
! 								oct_val = (oct_val << 3) + OCTVALUE(nextc);
! 								nextc = *(scan_end + 3);
! 								if (ISOCTAL(nextc))
  									{
! 									skip++;
! 									oct_val = (oct_val << 3) + OCTVALUE(nextc);
  									}
  								}
! 								newc = oct_val & 0377;	/* the escaped byte value */
  					break;
  				case 'x':
  					/* Handle \x3F */
! 							hexchar = *(scan_end + 2);
  
  						if (isxdigit(hexchar))
  						{
! 								skip++;
! 								hex_val = GetDecimalFromHex(hexchar);
  
! 								hexchar = *(scan_end + 3);
  								if (isxdigit(hexchar))
  								{
! 									skip++;
! 									hex_val = (hex_val << 4) + GetDecimalFromHex(hexchar);
  						}
+ 								newc = hex_val & 0xff;
  					}
+ 								else /* "\x" with no hex value */
+ 									newc = nextc;
  					break;
  				case 'b':
! 							newc = '\b';
  					break;
  				case 'f':
! 							newc = '\f';
  					break;
  				case 'n':
! 							newc = '\n';
  					break;
  				case 'r':
! 							newc = '\r';
  					break;
  				case 't':
! 							newc = '\t';
  					break;
  				case 'v':
! 							newc = '\v';
! 							break;
! 						default:
! 							if (nextc == delimc)
! 								newc = delimc;
! 							else if (nextc == escapec)
! 								newc = escapec;
! 							else
! 								/* no escape sequence, take next char literaly */
! 								newc = nextc;
  					break;
+ 					}
+ 					
+ 					/* update to current length, add escape and escaped chars  */
+ 					attr_pre_len += scan_end - scan_start + 2;
+ 					/* update to current length, escaped char */
+ 					attr_post_len += scan_end - scan_start + 1;
  
  					/*
! 					 * Need to get rid of the escape character. This is done by
! 					 * loading the chunk up to including the escape character
! 					 * into the attribute buffer. Then overwritting the backslash
! 					 * with the escaped sequence or char, and continuing to scan
! 					 * from *after* the char than is after the escape in line buf.
  					 */
+ 					*(attr_buf.data + attr_buf.len - 1) = newc;
+ 					line_buf.cursor = scan_end - line_buf.data + skip;
+ 					scan_start = scan_end + skip;
+ 					chunk_start = line_buf.cursor;
+ 					chunk_len = 0;
  			}
! 				
! 			}						/* end delimiter/backslash */
! 
! 		}							/* end line buffer scan. */
! 
! 	/*
! 	 * Replace all delimiters with NULL for string termination.
! 	 * NOTE: only delimiters (NOT necessarily all delimc) are replaced.
! 	 * Example (delimc = '|'):
! 	 * - Before:  f  1	|  f  \|  2  |	f  3
! 	 * - After :  f  1 \0  f   |  2 \0	f  3
! 	 */
! 	for (attribute = 1; attribute < num_phys_attrs; attribute++)
! 	{
! 		if (attr_offsets[attribute] != 0)
! 			*(attr_buf.data + attr_offsets[attribute] - 1) = '\0';
  	}
  
! }
! 
! 
! /*
!  * Read a binary attribute
!  */
! static Datum
! CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo, Oid typioparam,
! 						bool *isnull)
! {
! 	int32		fld_size;
! 	Datum		result;
! 	
! 	fld_size = CopyGetInt32();
! 	if (CopyGetEof())
! 		ereport(ERROR,
! 				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 				 errmsg("unexpected EOF in COPY data")));
! 	if (fld_size == -1)
! 	{
  		*isnull = true;
! 		return (Datum) 0;
! 	}
! 	if (fld_size < 0)
! 		ereport(ERROR,
! 				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 				 errmsg("invalid field size")));
  
! 	/* reset attr_buf to empty, and load raw data in it */
! 	attr_buf.len = 0;
! 	attr_buf.data[0] = '\0';
! 	attr_buf.cursor = 0;
! 	
! 	enlargeStringInfo(&attr_buf, fld_size);
! 	
! 	CopyGetData(attr_buf.data, fld_size);
! 	if (CopyGetEof())
! 		ereport(ERROR,
! 				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 				 errmsg("unexpected EOF in COPY data")));
  
+ 	attr_buf.len = fld_size;
+ 	attr_buf.data[fld_size] = '\0';
+ 	
+ 	/* Call the column type's binary input converter */
+ 	result = FunctionCall2(flinfo,
+ 						   PointerGetDatum(&attr_buf),
+ 						   ObjectIdGetDatum(typioparam));
+ 	
+ 	/* Trouble if it didn't eat the whole buffer */
+ 	if (attr_buf.cursor != attr_buf.len)
+ 		ereport(ERROR,
+ 				(errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
+ 				 errmsg("incorrect binary data format")));
+ 	
+ 	*isnull = false;
+ 	return result;
+ }
  
  /*
!  * Read all the attributes of the data line in CSV mode,
   * performing de-escaping as needed. Escaping does not follow the normal
   * PostgreSQL text mode, but instead "standard" (i.e. common) CSV usage.
   *
***************
*** 2448,2472 ****
   *
   * null_print is the null marker string.  Note that this is compared to
   * the pre-de-escaped input string (thus if it is quoted it is not a NULL).
-  *
-  * *result is set to indicate what terminated the read:
-  *		NORMAL_ATTR:	column delimiter
-  *		END_OF_LINE:	end of line
-  *		UNTERMINATED_FIELD no quote detected at end of a quoted field
-  *
-  * In any case, the string read up to the terminator (or end of file)
-  * is returned.
-  *
-  * *isnull is set true or false depending on whether the input matched
-  * the null marker.  Note that the caller cannot check this since the
-  * returned string will be the post-de-escaping equivalent, which may
-  * look the same as some valid data string.
   *----------
   */
! 
! static char *
! CopyReadAttributeCSV(const char *delim, const char *null_print, char *quote,
! 					 char *escape, CopyReadResult *result, bool *isnull)
  {
  	char		delimc = delim[0];
  	char		quotec = quote[0];
--- 2947,2958 ----
   *
   * null_print is the null marker string.  Note that this is compared to
   * the pre-de-escaped input string (thus if it is quoted it is not a NULL).
   *----------
   */
! static void
! CopyReadAttributesCSV(const char *delim, const char *null_print, char *quote,
! 					  char *escape, int null_print_len, char *nulls, List *attnumlist, 
! 					  int *attr_offsets, int num_phys_attrs, Form_pg_attribute *attr)
  {
  	char		delimc = delim[0];
  	char		quotec = quote[0];
***************
*** 2477,2502 ****
  	int			input_len;
  	bool		in_quote = false;
  	bool		saw_quote = false;
  
! 	/* reset attribute_buf to empty */
! 	attribute_buf.len = 0;
! 	attribute_buf.data[0] = '\0';
  
- 	/* set default status */
- 	*result = END_OF_LINE;
  
  	for (;;)
  	{
  		end_cursor = line_buf.cursor;
  		if (line_buf.cursor >= line_buf.len)
  			break;
  		c = line_buf.data[line_buf.cursor++];
  
  		/* unquoted field delimiter  */
  		if (!in_quote && c == delimc)
  		{
! 			*result = NORMAL_ATTR;
! 			break;
  		}
  
  		/* start of quoted field (or part of field) */
--- 2963,3043 ----
  	int			input_len;
  	bool		in_quote = false;
  	bool		saw_quote = false;
+ 	int			attnum;			/* attribute number being parsed */
+ 	int			m;				/* attribute index being parsed */
+ 	ListCell   *cur;			/* cursor to attribute list used for this COPY */
  
! 	/*
! 	 * init variables for attribute scan
! 	 */
! 	attr_buf.len = 0;
! 	attr_buf.data[0] = '\0';
! 	attr_buf.cursor = 0;
! 	
! 	cur = list_head(attnumlist);
! 	attnum = lfirst_int(cur);
! 	m = attnum - 1;
! 	input_len = 0;
  
  
  	for (;;)
  	{
  		end_cursor = line_buf.cursor;
+ 		
+ 		/* finished processing attributes in line */
  		if (line_buf.cursor >= line_buf.len)
+ 		{
+ 			/* check whether raw input matched null marker */
+ 			input_len = end_cursor - start_cursor;
+ 			if (!saw_quote && input_len == null_print_len &&
+ 				strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0)
+ 				nulls[m] = 'n';
+ 			else
+ 				nulls[m] = ' ';
+ 			
+ 			if (in_quote)
+ 				ereport(ERROR,
+ 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+ 						 errmsg("unterminated CSV quoted field")));
+ 			
+ 			if (lnext(cur) != NULL)
+ 				ereport(ERROR,
+ 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
+ 						 errmsg("missing data for column \"%s\"",
+ 								NameStr(attr[m + 1]->attname))));
  			break;
+ 		}
+ 		
  		c = line_buf.data[line_buf.cursor++];
  
  		/* unquoted field delimiter  */
  		if (!in_quote && c == delimc)
  		{
! 			/* check whether raw input matched null marker */
! 			input_len = end_cursor - start_cursor;
! 			if (!saw_quote && input_len == null_print_len &&
! 				strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0)
! 				nulls[m] = 'n';
! 			else
! 				nulls[m] = ' ';
!             
! 			/* terminate attr string with '\0' */
! 			appendStringInfoCharMacro(&attr_buf, '\0');
! 			attr_buf.cursor++;
! 			
! 			/* setup next attribute scan */
! 			cur = lnext(cur);
! 			
! 			if (cur == NULL)
! 				ereport(ERROR,
! 						(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 						 errmsg("extra data after last expected column")));
! 			
! 			attnum = lfirst_int(cur);
! 			m = attnum - 1;
! 			attr_offsets[m] = attr_buf.cursor;
! 			start_cursor = line_buf.cursor;
! 			continue;
  		}
  
  		/* start of quoted field (or part of field) */
***************
*** 2520,2527 ****
  
  				if (nextc == escapec || nextc == quotec)
  				{
! 					appendStringInfoCharMacro(&attribute_buf, nextc);
  					line_buf.cursor++;
  					continue;
  				}
  			}
--- 3061,3069 ----
  
  				if (nextc == escapec || nextc == quotec)
  				{
! 					appendStringInfoCharMacro(&attr_buf, nextc);
  					line_buf.cursor++;
+ 					attr_buf.cursor++;
  					continue;
  				}
  			}
***************
*** 2537,2616 ****
  			in_quote = false;
  			continue;
  		}
! 		appendStringInfoCharMacro(&attribute_buf, c);
  	}
  
- 	if (in_quote)
- 		*result = UNTERMINATED_FIELD;
- 
- 	/* check whether raw input matched null marker */
- 	input_len = end_cursor - start_cursor;
- 	if (!saw_quote && input_len == strlen(null_print) &&
- 		strncmp(&line_buf.data[start_cursor], null_print, input_len) == 0)
- 		*isnull = true;
- 	else
- 		*isnull = false;
- 
- 	return attribute_buf.data;
  }
  
  /*
!  * Read a binary attribute
   */
! static Datum
! CopyReadBinaryAttribute(int column_no, FmgrInfo *flinfo,
! 						Oid typioparam, int32 typmod,
  						bool *isnull)
  {
! 	int32		fld_size;
! 	Datum		result;
! 
! 	fld_size = CopyGetInt32();
! 	if (CopyGetEof())
! 		ereport(ERROR,
! 				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 				 errmsg("unexpected EOF in COPY data")));
! 	if (fld_size == -1)
! 	{
! 		*isnull = true;
! 		return (Datum) 0;
  	}
! 	if (fld_size < 0)
! 		ereport(ERROR,
! 				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
! 				 errmsg("invalid field size")));
  
! 	/* reset attribute_buf to empty, and load raw data in it */
! 	attribute_buf.len = 0;
! 	attribute_buf.data[0] = '\0';
! 	attribute_buf.cursor = 0;
  
! 	enlargeStringInfo(&attribute_buf, fld_size);
  
- 	CopyGetData(attribute_buf.data, fld_size);
- 	if (CopyGetEof())
- 		ereport(ERROR,
- 				(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
- 				 errmsg("unexpected EOF in COPY data")));
  
! 	attribute_buf.len = fld_size;
! 	attribute_buf.data[fld_size] = '\0';
  
! 	/* Call the column type's binary input converter */
! 	result = FunctionCall3(flinfo,
! 						   PointerGetDatum(&attribute_buf),
! 						   ObjectIdGetDatum(typioparam),
! 						   Int32GetDatum(typmod));
  
- 	/* Trouble if it didn't eat the whole buffer */
- 	if (attribute_buf.cursor != attribute_buf.len)
- 		ereport(ERROR,
- 				(errcode(ERRCODE_INVALID_BINARY_REPRESENTATION),
- 				 errmsg("incorrect binary data format")));
  
- 	*isnull = false;
- 	return result;
- }
  
  /*
   * Send text representation of one attribute, with conversion and escaping
--- 3079,3145 ----
  			in_quote = false;
  			continue;
  		}
! 		appendStringInfoCharMacro(&attr_buf, c);
! 		attr_buf.cursor++;
  	}
  
  }
  
  /*
!  * Read the first attribute. This is mainly used to maintain support
!  * for an OID column. All the rest of the columns will be read at once with
!  * CopyReadAttributesText.
   */
! static char *
! CopyReadOidAttr(const char *delim, const char *null_print, int null_print_len,
  						bool *isnull)
  {
! 	char		delimc = delim[0];
! 	char	   *start_loc = line_buf.data + line_buf.cursor;
! 	char	   *end_loc;
! 	int			attr_len = 0;
! 	int			bytes_remaining;
! 
! 	/* reset attribute buf to empty */
! 	attr_buf.len = 0;
! 	attr_buf.data[0] = '\0';
! 	attr_buf.cursor = 0;
! 
! 	/* # of bytes that were not yet processed in this line */
! 	bytes_remaining = line_buf.len - line_buf.cursor;
! 
! 	/* got to end of line */
! 	if ((end_loc = scanTextLine(start_loc, delimc, bytes_remaining)) == NULL)
! 	{
! 		attr_len = bytes_remaining - 1; /* don't count '\n' in len calculation */
! 		appendBinaryStringInfo(&attr_buf, start_loc, attr_len);
! 		line_buf.cursor += attr_len + 2;		/* skip '\n' and '\0' */
  	}
! 	else
! 		/* found a delimiter */
! 	{
! 		/*
! 		 * (we don't care if delim was preceded with a backslash, because it's
! 		 * an invalid OID anyway)
! 		 */
  
! 		attr_len = end_loc - start_loc; /* we don't include the delimiter ch */
  
! 		appendBinaryStringInfo(&attr_buf, start_loc, attr_len);
! 		line_buf.cursor += attr_len + 1;
! 	}
  
  
! 	/* check whether raw input matched null marker */
! 	if (attr_len == null_print_len && strncmp(start_loc, null_print, attr_len) == 0)
! 		*isnull = true;
! 	else
! 		*isnull = false;
  
! 	return attr_buf.data;
! }
  
  
  
  /*
   * Send text representation of one attribute, with conversion and escaping
***************
*** 2800,2802 ****
--- 3329,3432 ----
  
  	return attnums;
  }
+ 
+ /*
+  * The following are custom versions of the string function strchr().
+  * As opposed to the original strchr which searches through
+  * a string until the target character is found, or a NULL is
+  * found, this version will not return when a NULL is found.
+  * Instead it will search through a pre-defined length of
+  * bytes and will return only if the target character(s) is reached.
+  *
+  * If our client encoding is not a supported server encoding, we
+  * know that it is not safe to look at each character as trailing
+  * byte in a multibyte character may be a 7-bit ASCII equivalent.
+  * Therefore we use pg_encoding_mblen to skip to the end of the
+  * character.
+  *
+  * returns:
+  *	 pointer to c - if c is located within the string.
+  *	 NULL - if c was not found in specified length of search. Note:
+  *			this DOESN'T mean that a '\0' was reached.
+  */
+ char *
+ scanTextLine(const char *s, char eol, size_t len)
+ {
+ 	const char *start;
+ 
+ 	if (client_encoding_only && !line_buf_converted)
+ 	{
+ 		int			mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s);
+ 
+ 		for (start = s; *s != eol && s < start + len; s += mblen)
+ 			mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s);
+ 		
+ 		return ((*s == eol) ? (char *) s : NULL);
+ 	}
+ 	else
+ 		return memchr(s,eol,len);
+ }
+ 
+ 
+ char *
+ scanCSVLine(const char *s, char eol, char escapec, char quotec, size_t len)
+ {
+ 	const char *start;
+ 	
+ 	if (client_encoding_only && !line_buf_converted)
+ 	{
+ 		int			mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s);
+ 		
+ 		for (start = s; *s != eol &&  s < start + len; s += mblen)
+ 		{
+ 			if (in_quote && *s == escapec)
+ 				last_was_esc = !last_was_esc;
+ 			if (*s == quotec && !last_was_esc)
+ 				in_quote = !in_quote;
+ 			if (*s != escapec)
+ 				last_was_esc = false;
+ 					
+ 			mblen = pg_encoding_mblen(client_encoding, (unsigned char*)s);
+         }
+ 	}
+ 	else
+ 		/* safe to scroll byte by byte */
+ 	{
+ 		for (start = s; *s != eol  && s < start + len; s++)
+ 		{
+ 			if (in_quote && *s == escapec)
+ 				last_was_esc = !last_was_esc;
+ 			if (*s == quotec && !last_was_esc)
+ 				in_quote = !in_quote;
+ 			if (*s != escapec)
+ 				last_was_esc = false;
+ 		}
+ 			
+ 	}
+ 	
+ 	if(*s != escapec) /* found eol_ch */
+ 		last_was_esc = false;
+ 	
+ 	return ((*s == eol) ? (char *) s : NULL);
+ }
+ 
+ /*
+  * Scan for 1 of 2 characters necessary for attribute parsing. No need
+  * for checking for multi-byte characters since conversion already 
+  * happened.
+  */
+ char *
+ scanTextAttr(const char *s, char c1, char c2, size_t len)
+ {
+ 	const char *start;
+ 	
+ 	for (start = s; *s != c1 && *s != c2 && s < start + len; s++)
+ 		;
+ 	
+ 	return (*s != '\0' ? (char *) s : NULL);
+ }
+ 
+ 
+ 
+ 
+ 
