Patch for Improved Syntax Error Reporting

Attached please find a patch to the input parser that yields better
syntax error reporting on parse errors. For example:

test=# SELECT * FRUM bob;
ERROR: parser: parse error at or near "frum"

becomes:

test=# SELECT * FRUM bob;
ERROR: parser: parse error at or near 'frum':
SELECT * FRUM bob;
^

I've also modified the regression tests accordingly.

I haven't made the corresponding changes to the ecpg grammar -- I'm not
sure whether changes like this are desirable there. Feedback welcome.

Comments?

Neil

--
Neil Padgett
Red Hat Canada Ltd. E-Mail: npadgett(at)redhat(dot)com
2323 Yonge Street, Suite #300,
Toronto, ON M4P 2C9

Index: src/backend/parser/scan.l
===================================================================
RCS file: /home/projects/pgsql/cvsroot/pgsql/src/backend/parser/scan.l,v
retrieving revision 1.88
diff -c -p -r1.88 scan.l
*** src/backend/parser/scan.l 2001/03/22 17:41:47 1.88
--- src/backend/parser/scan.l 2001/08/01 17:43:53
***************
*** 37,42 ****
--- 37,43 ----

extern char *parseString;
static char *parseCh;
+ static int parseOffset;

/* some versions of lex define this as a macro */
#if defined(yywrap)
*************** other .
*** 254,262 ****
*/

%%
! {whitespace} { /* ignore */ }

! {xcstart} {
xcdepth = 0;
BEGIN(xc);
/* Put back any characters past slash-star; see above */
--- 255,266 ----
*/

%%
! {whitespace} { parseOffset += yyleng;
! /* ignore */
! }

! {xcstart} {
! parseOffset += 2;
xcdepth = 0;
BEGIN(xc);
/* Put back any characters past slash-star; see above */
*************** other .
*** 264,293 ****
}

<xc>{xcstart} {
xcdepth++;
/* Put back any characters past slash-star; see above */
yyless(2);
}

<xc>{xcstop} {
if (xcdepth <= 0)
BEGIN(INITIAL);
else
xcdepth--;
}

! <xc>{xcinside} { /* ignore */ }

! <xc>{op_chars} { /* ignore */ }

<xc><<EOF>> { elog(ERROR, "Unterminated /* comment"); }

! {xbitstart} {
BEGIN(xbit);
startlit();
addlit("b", 1);
}
<xbit>{xbitstop} {
BEGIN(INITIAL);
if (literalbuf[strspn(literalbuf + 1, "01") + 1] != '\0')
elog(ERROR, "invalid bit string input: '%s'",
--- 268,303 ----
}

<xc>{xcstart} {
+ parseOffset += 2;
xcdepth++;
/* Put back any characters past slash-star; see above */
yyless(2);
}

<xc>{xcstop} {
+ parseOffset += yyleng;
if (xcdepth <= 0)
BEGIN(INITIAL);
else
xcdepth--;
}

! <xc>{xcinside} { parseOffset += yyleng;
! /* ignore */ }

! <xc>{op_chars} { parseOffset += yyleng;
! /* ignore */ }

<xc><<EOF>> { elog(ERROR, "Unterminated /* comment"); }

! {xbitstart} {
! parseOffset += yyleng;
BEGIN(xbit);
startlit();
addlit("b", 1);
}
<xbit>{xbitstop} {
+ parseOffset += yyleng;
BEGIN(INITIAL);
if (literalbuf[strspn(literalbuf + 1, "01") + 1] != '\0')
elog(ERROR, "invalid bit string input: '%s'",
*************** other .
*** 297,311 ****
}
<xh>{xhinside} |
<xbit>{xbitinside} {
addlit(yytext, yyleng);
}
<xh>{xhcat} |
! <xbit>{xbitcat} {
/* ignore */
}
<xbit><<EOF>> { elog(ERROR, "unterminated bit string literal"); }

{xhstart} {
BEGIN(xh);
startlit();
}
--- 307,324 ----
}
<xh>{xhinside} |
<xbit>{xbitinside} {
+ parseOffset += yyleng;
addlit(yytext, yyleng);
}
<xh>{xhcat} |
! <xbit>{xbitcat} {
! parseOffset += yyleng;
/* ignore */
}
<xbit><<EOF>> { elog(ERROR, "unterminated bit string literal"); }

{xhstart} {
+ parseOffset += yyleng;
BEGIN(xh);
startlit();
}
*************** other .
*** 313,318 ****
--- 326,332 ----
long val;
char* endptr;

+ parseOffset += yyleng;
BEGIN(INITIAL);
errno = 0;
val = strtol(literalbuf, &endptr, 16);
*************** other .
*** 330,339 ****
--- 344,355 ----
<xh><<EOF>> { elog(ERROR, "Unterminated hexadecimal integer"); }

{xqstart} {
+ parseOffset += yyleng;
BEGIN(xq);
startlit();
}
<xq>{xqstop} {
+ parseOffset += yyleng;
BEGIN(INITIAL);
yylval.str = scanstr(literalbuf);
return SCONST;
*************** other .
*** 341,359 ****
--- 357,379 ----
<xq>{xqdouble} |
<xq>{xqinside} |
<xq>{xqliteral} {
+ parseOffset += yyleng;
addlit(yytext, yyleng);
}
<xq>{xqcat} {
+ parseOffset += yyleng;
/* ignore */
}
<xq><<EOF>> { elog(ERROR, "Unterminated quoted string"); }


{xdstart} {
+ parseOffset += yyleng;
BEGIN(xd);
startlit();
}
<xd>{xdstop} {
+ parseOffset += yyleng;
BEGIN(INITIAL);
if (strlen(literalbuf) == 0)
elog(ERROR, "zero-length delimited identifier");
*************** other .
*** 375,391 ****
return IDENT;
}
<xd>{xddouble} {
addlit(yytext, yyleng-1);
}
! <xd>{xdinside} {
addlit(yytext, yyleng);
}
<xd><<EOF>> { elog(ERROR, "Unterminated quoted identifier"); }

! {typecast} { return TYPECAST; }

- {self} { return yytext[0]; }
-
{operator} {
/*
* Check for embedded slash-star or dash-dash; those
--- 395,417 ----
return IDENT;
}
<xd>{xddouble} {
+ parseOffset += yyleng;
addlit(yytext, yyleng-1);
}
! <xd>{xdinside} {
! parseOffset += yyleng;
addlit(yytext, yyleng);
}
<xd><<EOF>> { elog(ERROR, "Unterminated quoted identifier"); }

! {typecast} {
! parseOffset += yyleng;
! return TYPECAST; }
!
! {self} {
! parseOffset += yyleng;
! return yytext[0]; }

{operator} {
/*
* Check for embedded slash-star or dash-dash; those
*************** other .
*** 396,401 ****
--- 422,429 ----
int nchars = yyleng;
char *slashstar = strstr((char*)yytext, "/*");
char *dashdash = strstr((char*)yytext, "--");
+
+ parseOffset += yyleng;

if (slashstar && dashdash)
{
*************** other .
*** 455,461 ****
return Op;
}

! {param} {
yylval.ival = atol((char*)&yytext[1]);
return PARAM;
}
--- 483,490 ----
return Op;
}

! {param}
{
! parseOffset += yyleng;
yylval.ival = atol((char*)&yytext[1]);
return PARAM;
}
*************** other .
*** 463,469 ****
{integer} {
long val;
char* endptr;
!
errno = 0;
val = strtol((char *)yytext, &endptr, 10);
if (*endptr != '\0' || errno == ERANGE
--- 492,499 ----
{integer} {
long val;
char* endptr;
!
! parseOffset += yyleng;
errno = 0;
val = strtol((char *)yytext, &endptr, 10);
if (*endptr != '\0' || errno == ERANGE
*************** other .
*** 480,490 ****
yylval.ival = val;
return ICONST;
}
! {decimal} {
yylval.str = pstrdup((char*)yytext);
return FCONST;
}
! {real} {
yylval.str = pstrdup((char*)yytext);
return FCONST;
}
--- 510,522 ----
yylval.ival = val;
return ICONST;
}
! {decimal}
{
! parseOffset += yyleng;
yylval.str = pstrdup((char*)yytext);
return FCONST;
}
! {real}
{
! parseOffset += yyleng;
yylval.str = pstrdup((char*)yytext);
return FCONST;
}
*************** other .
*** 493,498 ****
--- 525,532 ----
{identifier} {
ScanKeyword *keyword;
int i;
+
+ parseOffset += yyleng;

/* Is it a keyword? */
keyword = ScanKeywordLookup((char*) yytext);
*************** other .
*** 530,545 ****
return IDENT;
}

! {other} { return yytext[0]; }

%%

void
yyerror(const char *message)
{
! elog(ERROR, "parser: %s at or near \"%s\"", message, yytext);
}

int
yywrap(void)
{
--- 564,634 ----
return IDENT;
}

! {other} {
! parseOffset += yyleng;
! return yytext[0];
! }

%%

void
yyerror(const char *message)
{
! int errorOffset;
! char *line;
! char *endOfLine;
! char *beginningOfLine;
! size_t buffSize;
!
! /* Calculate the error's offset from the beginning of the input */
!
! errorOffset = parseOffset + 1 - yyleng;
!
! /* Find the beginning of the input line */
!
! for(beginningOfLine = parseString + errorOffset;
! beginningOfLine > parseString;
! beginningOfLine--)
! if(*(beginningOfLine - 1) == '\n' ||
! *(beginningOfLine - 1) == '\r' ||
! *(beginningOfLine - 1) == '\f')
! break;
!
! /* Find the end of the input line */
!
! for(endOfLine = parseString + errorOffset;
! *endOfLine != '\0';
! endOfLine++)
! if(*endOfLine == '\n' ||
! *endOfLine == '\r' ||
! *endOfLine == '\f')
! break;
!
! /* Calculate the offset of the error into the input line */
!
! errorOffset = errorOffset - (int)(beginningOfLine - parseString);
!
! /* Allocate a buffer for the line */
!
! buffSize = (endOfLine - beginningOfLine) + 1;
! line = palloc(buffSize);
!
! /* Copy the line into the buffer */
!
! memcpy(line, beginningOfLine, buffSize);
! *(line + buffSize - 1) = '\0';
!
! /* Report the error */
!
! elog(ERROR, "parser: %s at or near \"%s\":\n%s\n%*s",
! message,
! yytext,
! line,
! errorOffset,
! "^");
}

+
int
yywrap(void)
{
*************** scanner_init(void)
*** 557,562 ****
--- 646,655 ----
because input()/myinput() checks the non-nullness of parseCh
to know when to pass the string to lex/flex */
parseCh = NULL;
+
+ /* Initialize the parse input offset -- used by enhanced syntax error
reporting */
+
+ parseOffset = 0;

/* initialize literal buffer to a reasonable but expansible size */
literalalloc = 128;
Index: src/test/regress/expected/errors.out
===================================================================
RCS file:
/home/projects/pgsql/cvsroot/pgsql/src/test/regress/expected/errors.out,v
retrieving revision 1.26
diff -c -p -r1.26 errors.out
*** src/test/regress/expected/errors.out 2000/11/08 22:10:03 1.26
--- src/test/regress/expected/errors.out 2001/08/01 17:43:56
*************** select 1
*** 19,25 ****
select
-- no such relation
select * from nonesuch;
! ERROR: parser: parse error at or near "select"
-- bad name in target list
select nonesuch from pg_database;
ERROR: Attribute 'nonesuch' not found
--- 19,27 ----
select
-- no such relation
select * from nonesuch;
! ERROR: parser: parse error at or near "select":
! select
! ^
-- bad name in target list
select nonesuch from pg_database;
ERROR: Attribute 'nonesuch' not found
*************** select * from pg_database where pg_datab
*** 31,37 ****
ERROR: Attribute 'nonesuch' not found
-- bad select distinct on syntax, distinct attribute missing
select distinct on (foobar) from pg_database;
! ERROR: parser: parse error at or near "from"
-- bad select distinct on syntax, distinct attribute not in target
list
select distinct on (foobar) * from pg_database;
ERROR: Attribute 'foobar' not found
--- 33,41 ----
ERROR: Attribute 'nonesuch' not found
-- bad select distinct on syntax, distinct attribute missing
select distinct on (foobar) from pg_database;
! ERROR: parser: parse error at or near "from":
! select distinct on (foobar) from pg_database;
! ^
-- bad select distinct on syntax, distinct attribute not in target
list
select distinct on (foobar) * from pg_database;
ERROR: Attribute 'foobar' not found
*************** ERROR: Attribute 'foobar' not found
*** 39,46 ****
-- DELETE

-- missing relation name (this had better not wildcard!)
delete from;
! ERROR: parser: parse error at or near ";"
-- no such relation
delete from nonesuch;
ERROR: Relation 'nonesuch' does not exist
--- 43,52 ----
-- DELETE

-- missing relation name (this had better not wildcard!)
+ delete from;
+ ERROR: parser: parse error at or near ";":
delete from;
! ^
-- no such relation
delete from nonesuch;
ERROR: Relation 'nonesuch' does not exist
*************** ERROR: Relation 'nonesuch' does not exi
*** 49,55 ****

-- missing relation name (this had better not wildcard!)
drop table;
! ERROR: parser: parse error at or near ";"
-- no such relation
drop table nonesuch;
ERROR: table "nonesuch" does not exist
--- 55,63 ----

-- missing relation name (this had better not wildcard!)
drop table;
! ERROR: parser: parse error at or near ";":
! drop table;
! ^
-- no such relation
drop table nonesuch;
ERROR: table "nonesuch" does not exist
*************** ERROR: table "nonesuch" does not exist
*** 58,65 ****

-- relation renaming
-- missing relation name
alter table rename;
! ERROR: parser: parse error at or near ";"
-- no such relation
alter table nonesuch rename to newnonesuch;
ERROR: Relation "nonesuch" does not exist
--- 66,75 ----

-- relation renaming
-- missing relation name
+ alter table rename;
+ ERROR: parser: parse error at or near ";":
alter table rename;
! ^
-- no such relation
alter table nonesuch rename to newnonesuch;
ERROR: Relation "nonesuch" does not exist
*************** ERROR: Define: "basetype" unspecified
*** 116,125 ****

-- missing index name
drop index;
! ERROR: parser: parse error at or near ";"
-- bad index name
drop index 314159;
! ERROR: parser: parse error at or near "314159"
-- no such index
drop index nonesuch;
ERROR: index "nonesuch" does not exist
--- 126,139 ----

-- missing index name
drop index;
! ERROR: parser: parse error at or near ";":
! drop index;
! ^
-- bad index name
+ drop index 314159;
+ ERROR: parser: parse error at or near "314159":
drop index 314159;
! ^
-- no such index
drop index nonesuch;
ERROR: index "nonesuch" does not exist
*************** ERROR: index "nonesuch" does not exist
*** 127,143 ****
-- REMOVE AGGREGATE

-- missing aggregate name
drop aggregate;
! ERROR: parser: parse error at or near ";"
-- bad aggregate name
drop aggregate 314159;
! ERROR: parser: parse error at or near "314159"
-- no such aggregate
drop aggregate nonesuch;
! ERROR: parser: parse error at or near ";"
-- missing aggregate type
drop aggregate newcnt1;
! ERROR: parser: parse error at or near ";"
-- bad aggregate type
drop aggregate newcnt nonesuch;
ERROR: RemoveAggregate: type 'nonesuch' does not exist
--- 141,165 ----
-- REMOVE AGGREGATE

-- missing aggregate name
+ drop aggregate;
+ ERROR: parser: parse error at or near ";":
drop aggregate;
! ^
-- bad aggregate name
drop aggregate 314159;
! ERROR: parser: parse error at or near "314159":
! drop aggregate 314159;
! ^
-- no such aggregate
+ drop aggregate nonesuch;
+ ERROR: parser: parse error at or near ";":
drop aggregate nonesuch;
! ^
-- missing aggregate type
drop aggregate newcnt1;
! ERROR: parser: parse error at or near ";":
! drop aggregate newcnt1;
! ^
-- bad aggregate type
drop aggregate newcnt nonesuch;
ERROR: RemoveAggregate: type 'nonesuch' does not exist
*************** ERROR: RemoveAggregate: aggregate 'newc
*** 148,158 ****
-- REMOVE FUNCTION

-- missing function name
drop function ();
! ERROR: parser: parse error at or near "("
-- bad function name
drop function 314159();
! ERROR: parser: parse error at or near "314159"
-- no such function
drop function nonesuch();
ERROR: RemoveFunction: function 'nonesuch()' does not exist
--- 170,184 ----
-- REMOVE FUNCTION

-- missing function name
+ drop function ();
+ ERROR: parser: parse error at or near "(":
drop function ();
! ^
-- bad function name
drop function 314159();
! ERROR: parser: parse error at or near "314159":
! drop function 314159();
! ^
-- no such function
drop function nonesuch();
ERROR: RemoveFunction: function 'nonesuch()' does not exist
*************** ERROR: RemoveFunction: function 'nonesu
*** 160,170 ****
-- REMOVE TYPE

-- missing type name
drop type;
! ERROR: parser: parse error at or near ";"
-- bad type name
drop type 314159;
! ERROR: parser: parse error at or near "314159"
-- no such type
drop type nonesuch;
ERROR: RemoveType: type 'nonesuch' does not exist
--- 186,200 ----
-- REMOVE TYPE

-- missing type name
+ drop type;
+ ERROR: parser: parse error at or near ";":
drop type;
! ^
-- bad type name
+ drop type 314159;
+ ERROR: parser: parse error at or near "314159":
drop type 314159;
! ^
-- no such type
drop type nonesuch;
ERROR: RemoveType: type 'nonesuch' does not exist
*************** ERROR: RemoveType: type 'nonesuch' does
*** 173,194 ****

-- missing everything
drop operator;
! ERROR: parser: parse error at or near ";"
-- bad operator name
drop operator equals;
! ERROR: parser: parse error at or near "equals"
-- missing type list
drop operator ===;
! ERROR: parser: parse error at or near ";"
-- missing parentheses
drop operator int4, int4;
! ERROR: parser: parse error at or near "int4"
-- missing operator name
drop operator (int4, int4);
! ERROR: parser: parse error at or near "("
-- missing type list contents
drop operator === ();
! ERROR: parser: parse error at or near ")"
-- no such operator
drop operator === (int4);
ERROR: parser: argument type missing (use NONE for unary operators)
--- 203,236 ----

-- missing everything
drop operator;
! ERROR: parser: parse error at or near ";":
! drop operator;
! ^
-- bad operator name
+ drop operator equals;
+ ERROR: parser: parse error at or near "equals":
drop operator equals;
! ^
-- missing type list
drop operator ===;
! ERROR: parser: parse error at or near ";":
! drop operator ===;
! ^
-- missing parentheses
+ drop operator int4, int4;
+ ERROR: parser: parse error at or near "int4":
drop operator int4, int4;
! ^
-- missing operator name
drop operator (int4, int4);
! ERROR: parser: parse error at or near "(":
! drop operator (int4, int4);
! ^
-- missing type list contents
+ drop operator === ();
+ ERROR: parser: parse error at or near ")":
drop operator === ();
! ^
-- no such operator
drop operator === (int4);
ERROR: parser: argument type missing (use NONE for unary operators)
*************** ERROR: RemoveOperator: binary operator
*** 199,206 ****
drop operator = (nonesuch);
ERROR: parser: argument type missing (use NONE for unary operators)
-- no such type1
drop operator = ( , int4);
! ERROR: parser: parse error at or near ","
-- no such type1
drop operator = (nonesuch, int4);
ERROR: RemoveOperator: type 'nonesuch' does not exist
--- 241,250 ----
drop operator = (nonesuch);
ERROR: parser: argument type missing (use NONE for unary operators)
-- no such type1
+ drop operator = ( , int4);
+ ERROR: parser: parse error at or near ",":
drop operator = ( , int4);
! ^
-- no such type1
drop operator = (nonesuch, int4);
ERROR: RemoveOperator: type 'nonesuch' does not exist
*************** drop operator = (int4, nonesuch);
*** 209,233 ****
ERROR: RemoveOperator: type 'nonesuch' does not exist
-- no such type2
drop operator = (int4, );
! ERROR: parser: parse error at or near ")"
--
-- DROP RULE

-- missing rule name
drop rule;
! ERROR: parser: parse error at or near ";"
-- bad rule name
drop rule 314159;
! ERROR: parser: parse error at or near "314159"
-- no such rule
drop rule nonesuch;
ERROR: Rule or view "nonesuch" not found
-- bad keyword
drop tuple rule nonesuch;
! ERROR: parser: parse error at or near "tuple"
-- no such rule
drop instance rule nonesuch;
! ERROR: parser: parse error at or near "instance"
-- no such rule
drop rewrite rule nonesuch;
! ERROR: parser: parse error at or near "rewrite"
--- 253,289 ----
ERROR: RemoveOperator: type 'nonesuch' does not exist
-- no such type2
drop operator = (int4, );
! ERROR: parser: parse error at or near ")":
! drop operator = (int4, );
! ^
--
-- DROP RULE

-- missing rule name
+ drop rule;
+ ERROR: parser: parse error at or near ";":
drop rule;
! ^
-- bad rule name
+ drop rule 314159;
+ ERROR: parser: parse error at or near "314159":
drop rule 314159;
! ^
-- no such rule
drop rule nonesuch;
ERROR: Rule or view "nonesuch" not found
-- bad keyword
drop tuple rule nonesuch;
! ERROR: parser: parse error at or near "tuple":
! drop tuple rule nonesuch;
! ^
-- no such rule
+ drop instance rule nonesuch;
+ ERROR: parser: parse error at or near "instance":
drop instance rule nonesuch;
! ^
-- no such rule
+ drop rewrite rule nonesuch;
+ ERROR: parser: parse error at or near "rewrite":
drop rewrite rule nonesuch;
! ^
\ No newline at end of file
Index: src/test/regress/expected/strings.out
===================================================================
RCS file:
/home/projects/pgsql/cvsroot/pgsql/src/test/regress/expected/strings.out,v
retrieving revision 1.10
diff -c -p -r1.10 strings.out
*** src/test/regress/expected/strings.out 2001/06/01 17:49:17 1.10
--- src/test/regress/expected/strings.out 2001/08/01 17:43:56
*************** SELECT 'first line'
*** 17,23 ****
' - next line' /* this comment is not allowed here */
' - third line'
AS "Illegal comment within continuation";
! ERROR: parser: parse error at or near "'"
--
-- test conversions between various string types
--
--- 17,25 ----
' - next line' /* this comment is not allowed here */
' - third line'
AS "Illegal comment within continuation";
! ERROR: parser: parse error at or near "'":
! ' - third line'
! ^
--
-- test conversions between various string types
--
Index: src/test/regress/output/constraints.source
===================================================================
RCS file:
/home/projects/pgsql/cvsroot/pgsql/src/test/regress/output/constraints.source,v
retrieving revision 1.18
diff -c -p -r1.18 constraints.source
*** src/test/regress/output/constraints.source 2001/02/22 05:32:56 1.18
--- src/test/regress/output/constraints.source 2001/08/01 17:43:56
*************** SELECT '' AS four, * FROM DEFAULTEXPR_TB
*** 45,56 ****
-- syntax errors
-- test for extraneous comma
CREATE TABLE error_tbl (i int DEFAULT (100, ));
! ERROR: parser: parse error at or near ","
-- this will fail because gram.y uses b_expr not a_expr for defaults,
-- to avoid a shift/reduce conflict that arises from NOT NULL being
-- part of the column definition syntax:
CREATE TABLE error_tbl (b1 bool DEFAULT 1 IN (1, 2));
! ERROR: parser: parse error at or near "IN"
-- this should work, however:
CREATE TABLE error_tbl (b1 bool DEFAULT (1 IN (1, 2)));
DROP TABLE error_tbl;
--- 45,60 ----
-- syntax errors
-- test for extraneous comma
CREATE TABLE error_tbl (i int DEFAULT (100, ));
! ERROR: parser: parse error at or near ",":
! CREATE TABLE error_tbl (i int DEFAULT (100, ));
! ^
-- this will fail because gram.y uses b_expr not a_expr for defaults,
-- to avoid a shift/reduce conflict that arises from NOT NULL being
-- part of the column definition syntax:
CREATE TABLE error_tbl (b1 bool DEFAULT 1 IN (1, 2));
! ERROR: parser: parse error at or near "IN":
! CREATE TABLE error_tbl (b1 bool DEFAULT 1 IN (1, 2));
! ^
-- this should work, however:
CREATE TABLE error_tbl (b1 bool DEFAULT (1 IN (1, 2)));
DROP TABLE error_tbl;