This patch series changes several parsers in the backend and contrib
modules to use bison pure parsers and flex reentrant scanners. This is
ultimately toward thread-safety, but I think it's also just nicer in
general, and it might also fix a few possible small memory leaks.
I organized this patch series into very small incremental changes so
that it's easier to follow. The final commits should probably combined a
bit more (e.g., one per module).
In this patch series I have so far dealt with
* contrib/cube/
* contrib/seg/
* src/backend/replication/repl_*
* src/backend/replication/syncrep_*
These four needed the whole treatment: pure parser, reentrant scanner,
and updated memory handling.
Also:
* src/backend/utils/adt/jsonpath_scan.l
This one already had a pure parser and palloc-based memory handling, but
not a reentrant scanner, so I just did that.
The above are all pretty similar, so it was relatively easy to work
through them once I had the first one figured out.
A couple of things that are still missing in the above:
* For repl_scanner.l, I want to use yyextra to deal with the static
variables marked /*FIXME*/, but somehow I made that buggy, I'll need to
take another look later.
* For both the replication parser and the syncrep parser, get rid of the
global variables to pass back the results. Again, here I need another
look later. I confused either myself or the compiler on these.
cube, seg, and jsonpath are about as done as I would want them to be.
Not done yet:
* src/backend/utils/misc/guc-file.l
* src/pl/plpgsql/src/pl_gram.y
These have quite different structures and requirements, so I plan to
deal with them separately.
Not relevant for backend thread-safety:
* src/backend/bootstrap/
It might make sense to eventually covert that one as well, just so that
the APIs are kept similar. But that could be for later.
Note that the core scanner and parser are already reentrant+pure.
Also, there are various other scanners and parsers in frontends (psql,
pgbench, ecpg) that are not relevant for this. (Again, it might make
sense to convert some of them later, and some of them are already done.)
AFAICT, all the options and coding techniques used here are already in
use elsewhere in the tree, so there shouldn't be any concerns about
bison or flex version compatibility.
From 01823a7c975fda47ce220db5bb91ecd0959d5123 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Mon, 2 Dec 2024 10:35:37 +0100
Subject: [PATCH v0 01/15] cube: pure parser and reentrant scanner
Use the flex %option reentrant and the bison option %pure-parser to
make the generated scanner and parser pure, reentrant, and
thread-safe.
(There are still some issues in the surrounding integration, see
FIXMEs.)
---
contrib/cube/cube.c | 7 +++---
contrib/cube/cubedata.h | 15 ++++++++----
contrib/cube/cubeparse.y | 15 +++++-------
contrib/cube/cubescan.l | 51 +++++++++++++++++++---------------------
4 files changed, 45 insertions(+), 43 deletions(-)
diff --git a/contrib/cube/cube.c b/contrib/cube/cube.c
index 1fc447511a1..bf8fc489dca 100644
--- a/contrib/cube/cube.c
+++ b/contrib/cube/cube.c
@@ -120,13 +120,14 @@ cube_in(PG_FUNCTION_ARGS)
char *str = PG_GETARG_CSTRING(0);
NDBOX *result;
Size scanbuflen;
+ yyscan_t scanner;
- cube_scanner_init(str, &scanbuflen);
+ cube_scanner_init(str, &scanbuflen, &scanner);
- cube_yyparse(&result, scanbuflen, fcinfo->context);
+ cube_yyparse(&result, scanbuflen, fcinfo->context, scanner);
/* We might as well run this even on failure. */
- cube_scanner_finish();
+ cube_scanner_finish(scanner);
PG_RETURN_NDBOX_P(result);
}
diff --git a/contrib/cube/cubedata.h b/contrib/cube/cubedata.h
index 96fa41a04e7..8bfcc6e99a2 100644
--- a/contrib/cube/cubedata.h
+++ b/contrib/cube/cubedata.h
@@ -59,14 +59,21 @@ typedef struct NDBOX
#define CubeKNNDistanceEuclid 17 /* <-> */
#define CubeKNNDistanceChebyshev 18 /* <=> */
+/* for cubescan.l and cubeparse.y */
+/* All grammar constructs return strings */
+#define YYSTYPE char *
+typedef void *yyscan_t;
+
/* in cubescan.l */
-extern int cube_yylex(void);
+extern int cube_yylex(YYSTYPE *yylval_param, yyscan_t yyscanner);
extern void cube_yyerror(NDBOX **result, Size scanbuflen,
struct Node *escontext,
+ yyscan_t yyscanner,
const char *message);
-extern void cube_scanner_init(const char *str, Size *scanbuflen);
-extern void cube_scanner_finish(void);
+extern void cube_scanner_init(const char *str, Size *scanbuflen, yyscan_t
*yyscannerp);
+extern void cube_scanner_finish(yyscan_t yyscanner);
/* in cubeparse.y */
extern int cube_yyparse(NDBOX **result, Size scanbuflen,
- struct Node *escontext);
+ struct Node *escontext,
+ yyscan_t yyscanner);
diff --git a/contrib/cube/cubeparse.y b/contrib/cube/cubeparse.y
index 52622875cbb..a6b7e70630d 100644
--- a/contrib/cube/cubeparse.y
+++ b/contrib/cube/cubeparse.y
@@ -7,19 +7,11 @@
#include "postgres.h"
#include "cubedata.h"
+#include "cubeparse.h" /* must be after cubedata.h for YYSTYPE and NDBOX */
#include "nodes/miscnodes.h"
#include "utils/float.h"
#include "varatt.h"
-/* All grammar constructs return strings */
-#define YYSTYPE char *
-
-#include "cubeparse.h"
-
-/* silence -Wmissing-variable-declarations */
-extern int cube_yychar;
-extern int cube_yynerrs;
-
/*
* Bison doesn't allocate anything that needs to live across parser calls,
* so we can easily have it use palloc instead of malloc. This prevents
@@ -40,6 +32,9 @@ static bool write_point_as_box(int dim, char *str,
%parse-param {NDBOX **result}
%parse-param {Size scanbuflen}
%parse-param {struct Node *escontext}
+%parse-param {yyscan_t yyscanner}
+%lex-param {yyscan_t yyscanner}
+%pure-parser
%expect 0
%name-prefix="cube_yy"
@@ -75,6 +70,8 @@ box: O_BRACKET paren_list COMMA paren_list C_BRACKET
if (!write_box(dim, $2, $4, result, escontext))
YYABORT;
+
+ (void) yynerrs; /* suppress compiler warning */
}
| paren_list COMMA paren_list
diff --git a/contrib/cube/cubescan.l b/contrib/cube/cubescan.l
index a30fbfc3111..09109675711 100644
--- a/contrib/cube/cubescan.l
+++ b/contrib/cube/cubescan.l
@@ -6,13 +6,8 @@
#include "postgres.h"
-/*
- * NB: include cubeparse.h only AFTER defining YYSTYPE (to match cubeparse.y)
- * and cubedata.h for NDBOX.
- */
#include "cubedata.h"
-#define YYSTYPE char *
-#include "cubeparse.h"
+#include "cubeparse.h" /* must be after cubedata.h for YYSTYPE and NDBOX */
}
%{
@@ -32,10 +27,11 @@ fprintf_to_ereport(const char *fmt, const char *msg)
}
/* Handles to the buffer that the lexer uses internally */
-static YY_BUFFER_STATE scanbufhandle;
-static char *scanbuf;
+static char *scanbuf; // FIXME
%}
+%option reentrant
+%option bison-bridge
%option 8bit
%option never-interactive
%option nodefault
@@ -55,14 +51,14 @@ NaN [nN][aA][nN]
%%
-{float} cube_yylval = yytext; return CUBEFLOAT;
-{infinity} cube_yylval = yytext; return CUBEFLOAT;
-{NaN} cube_yylval = yytext; return CUBEFLOAT;
-\[ cube_yylval = "("; return O_BRACKET;
-\] cube_yylval = ")"; return C_BRACKET;
-\( cube_yylval = "("; return O_PAREN;
-\) cube_yylval = ")"; return C_PAREN;
-\, cube_yylval = ","; return COMMA;
+{float} *yylval = yytext; return CUBEFLOAT;
+{infinity} *yylval = yytext; return CUBEFLOAT;
+{NaN} *yylval = yytext; return CUBEFLOAT;
+\[ *yylval = "("; return O_BRACKET;
+\] *yylval = ")"; return C_BRACKET;
+\( *yylval = "("; return O_PAREN;
+\) *yylval = ")"; return C_PAREN;
+\, *yylval = ","; return COMMA;
[ \t\n\r\f\v]+ /* discard spaces */
. return yytext[0]; /* alert parser of the garbage */
@@ -74,8 +70,11 @@ NaN [nN][aA][nN]
void
cube_yyerror(NDBOX **result, Size scanbuflen,
struct Node *escontext,
+ yyscan_t yyscanner,
const char *message)
{
+ struct yyguts_t * yyg = (struct yyguts_t *) yyscanner; /* needed for
yytext macro */
+
if (*yytext == YY_END_OF_BUFFER_CHAR)
{
errsave(escontext,
@@ -99,15 +98,15 @@ cube_yyerror(NDBOX **result, Size scanbuflen,
* Called before any actual parsing is done
*/
void
-cube_scanner_init(const char *str, Size *scanbuflen)
+cube_scanner_init(const char *str, Size *scanbuflen, yyscan_t *yyscannerp)
{
Size slen = strlen(str);
+ yyscan_t yyscanner;
- /*
- * Might be left over after ereport()
- */
- if (YY_CURRENT_BUFFER)
- yy_delete_buffer(YY_CURRENT_BUFFER);
+ if (yylex_init(yyscannerp) != 0)
+ elog(ERROR, "yylex_init() failed: %m");
+
+ yyscanner = *yyscannerp;
/*
* Make a scan buffer with special termination needed by flex.
@@ -116,9 +115,7 @@ cube_scanner_init(const char *str, Size *scanbuflen)
scanbuf = palloc(slen + 2);
memcpy(scanbuf, str, slen);
scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
- scanbufhandle = yy_scan_buffer(scanbuf, slen + 2);
-
- BEGIN(INITIAL);
+ yy_scan_buffer(scanbuf, slen + 2, yyscanner);
}
@@ -126,8 +123,8 @@ cube_scanner_init(const char *str, Size *scanbuflen)
* Called after parsing is done to clean up after cube_scanner_init()
*/
void
-cube_scanner_finish(void)
+cube_scanner_finish(yyscan_t yyscanner)
{
- yy_delete_buffer(scanbufhandle);
+ yylex_destroy(yyscanner);
pfree(scanbuf);
}
base-commit: 2f696453d2b39fea800d5f7d8e5d3e1a2266de24
--
2.47.1
From f092720cf73b62194d10ddf65948d03bbaafb04b Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Mon, 2 Dec 2024 10:35:37 +0100
Subject: [PATCH v0 02/15] cube: Use palloc() instead of malloc() for flex
Make the generated scanner use palloc() etc. instead of malloc() etc.
Previously, we only used palloc() for the buffer, but flex would still
use malloc() for its internal structures. As a result, there could be
some small memory leaks in case of uncaught errors. (We do catch
normal syntax errors as soft errors.) Now, all the memory is under
palloc() control, so there are no more such issues.
---
contrib/cube/cubescan.l | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/contrib/cube/cubescan.l b/contrib/cube/cubescan.l
index 09109675711..b741e77d7e1 100644
--- a/contrib/cube/cubescan.l
+++ b/contrib/cube/cubescan.l
@@ -38,6 +38,9 @@ static char *scanbuf; // FIXME
%option noinput
%option nounput
%option noyywrap
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
%option warn
%option prefix="cube_yy"
@@ -128,3 +131,30 @@ cube_scanner_finish(yyscan_t yyscanner)
yylex_destroy(yyscanner);
pfree(scanbuf);
}
+
+/*
+ * Interface functions to make flex use palloc() instead of malloc().
+ * It'd be better to make these static, but flex insists otherwise.
+ */
+
+void *
+yyalloc(yy_size_t size, yyscan_t yyscanner)
+{
+ return palloc(size);
+}
+
+void *
+yyrealloc(void *ptr, yy_size_t size, yyscan_t yyscanner)
+{
+ if (ptr)
+ return repalloc(ptr, size);
+ else
+ return palloc(size);
+}
+
+void
+yyfree(void *ptr, yyscan_t yyscanner)
+{
+ if (ptr)
+ pfree(ptr);
+}
--
2.47.1
From 0913702c2605fb437c8f140325283006877f11b3 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Mon, 2 Dec 2024 10:35:37 +0100
Subject: [PATCH v0 03/15] cube: Simplify flex scan buffer management
Instead of constructing the buffer from pieces and then using
yy_scan_buffer(), we can just use yy_scan_string(), which does the
same thing internally. (Actually, we use yy_scan_bytes() here because
we already have the length.)
The previous code was necessary because we allocated the buffer with
palloc() and the rest of the state was handled by malloc(). But this
is no longer the case; everything is under palloc() now.
XXX We could even get rid of the yylex_destroy() call and just let the
memory context cleanup handle everything.
---
contrib/cube/cubescan.l | 12 +-----------
1 file changed, 1 insertion(+), 11 deletions(-)
diff --git a/contrib/cube/cubescan.l b/contrib/cube/cubescan.l
index b741e77d7e1..eed324d6e3b 100644
--- a/contrib/cube/cubescan.l
+++ b/contrib/cube/cubescan.l
@@ -25,9 +25,6 @@ fprintf_to_ereport(const char *fmt, const char *msg)
{
ereport(ERROR, (errmsg_internal("%s", msg)));
}
-
-/* Handles to the buffer that the lexer uses internally */
-static char *scanbuf; // FIXME
%}
%option reentrant
@@ -111,14 +108,8 @@ cube_scanner_init(const char *str, Size *scanbuflen,
yyscan_t *yyscannerp)
yyscanner = *yyscannerp;
- /*
- * Make a scan buffer with special termination needed by flex.
- */
+ yy_scan_bytes(str, slen, yyscanner);
*scanbuflen = slen;
- scanbuf = palloc(slen + 2);
- memcpy(scanbuf, str, slen);
- scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
- yy_scan_buffer(scanbuf, slen + 2, yyscanner);
}
@@ -129,7 +120,6 @@ void
cube_scanner_finish(yyscan_t yyscanner)
{
yylex_destroy(yyscanner);
- pfree(scanbuf);
}
/*
--
2.47.1
From 7a4cd3ffee3009a5f6f5cbfd2fed362e6b87ad2c Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Mon, 2 Dec 2024 10:35:37 +0100
Subject: [PATCH v0 04/15] seg: pure parser and reentrant scanner
---
contrib/seg/seg.c | 9 ++++----
contrib/seg/segdata.h | 13 ++++++++----
contrib/seg/segparse.y | 9 ++++----
contrib/seg/segscan.l | 47 ++++++++++++++++++++----------------------
4 files changed, 41 insertions(+), 37 deletions(-)
diff --git a/contrib/seg/seg.c b/contrib/seg/seg.c
index 7f9fc24eb4b..fd4216edc5d 100644
--- a/contrib/seg/seg.c
+++ b/contrib/seg/seg.c
@@ -105,13 +105,14 @@ seg_in(PG_FUNCTION_ARGS)
{
char *str = PG_GETARG_CSTRING(0);
SEG *result = palloc(sizeof(SEG));
+ yyscan_t scanner;
- seg_scanner_init(str);
+ seg_scanner_init(str, &scanner);
- if (seg_yyparse(result, fcinfo->context) != 0)
- seg_yyerror(result, fcinfo->context, "bogus input");
+ if (seg_yyparse(result, fcinfo->context, scanner) != 0)
+ seg_yyerror(result, fcinfo->context, scanner, "bogus input");
- seg_scanner_finish();
+ seg_scanner_finish(scanner);
PG_RETURN_POINTER(result);
}
diff --git a/contrib/seg/segdata.h b/contrib/seg/segdata.h
index 3d6e3e3f245..7bc7c83dca3 100644
--- a/contrib/seg/segdata.h
+++ b/contrib/seg/segdata.h
@@ -14,12 +14,17 @@ typedef struct SEG
/* in seg.c */
extern int significant_digits(const char *s);
+/* for segscan.l and segparse.y */
+union YYSTYPE;
+typedef void *yyscan_t;
+
/* in segscan.l */
-extern int seg_yylex(void);
+extern int seg_yylex(union YYSTYPE *yylval_param, yyscan_t yyscanner);
extern void seg_yyerror(SEG *result, struct Node *escontext,
+ yyscan_t yyscanner,
const char *message);
-extern void seg_scanner_init(const char *str);
-extern void seg_scanner_finish(void);
+extern void seg_scanner_init(const char *str, yyscan_t *yyscannerp);
+extern void seg_scanner_finish(yyscan_t yyscanner);
/* in segparse.y */
-extern int seg_yyparse(SEG *result, struct Node *escontext);
+extern int seg_yyparse(SEG *result, struct Node *escontext, yyscan_t
yyscanner);
diff --git a/contrib/seg/segparse.y b/contrib/seg/segparse.y
index 3e4aa31cada..0358ddb182c 100644
--- a/contrib/seg/segparse.y
+++ b/contrib/seg/segparse.y
@@ -14,10 +14,6 @@
#include "segdata.h"
#include "segparse.h"
-/* silence -Wmissing-variable-declarations */
-extern int seg_yychar;
-extern int seg_yynerrs;
-
/*
* Bison doesn't allocate anything that needs to live across parser calls,
* so we can easily have it use palloc instead of malloc. This prevents
@@ -35,6 +31,9 @@ static int sig_digits(const char *value);
/* BISON Declarations */
%parse-param {SEG *result}
%parse-param {struct Node *escontext}
+%parse-param {yyscan_t yyscanner}
+%lex-param {yyscan_t yyscanner}
+%pure-parser
%expect 0
%name-prefix="seg_yy"
@@ -72,6 +71,8 @@ range: boundary PLUMIN deviation
result->u_sigd = Max(sig_digits(strbuf), Max($1.sigd, $3.sigd));
result->l_ext = '\0';
result->u_ext = '\0';
+
+ (void) yynerrs; /* suppress compiler warning */
}
| boundary RANGE boundary
diff --git a/contrib/seg/segscan.l b/contrib/seg/segscan.l
index 4ad529eccc4..35c3a11ac14 100644
--- a/contrib/seg/segscan.l
+++ b/contrib/seg/segscan.l
@@ -6,12 +6,8 @@
#include "nodes/miscnodes.h"
-/*
- * NB: include segparse.h only AFTER including segdata.h, because segdata.h
- * contains the definition for SEG.
- */
#include "segdata.h"
-#include "segparse.h"
+#include "segparse.h" /* must be after segdata.h SEG */
}
%{
@@ -31,10 +27,11 @@ fprintf_to_ereport(const char *fmt, const char *msg)
}
/* Handles to the buffer that the lexer uses internally */
-static YY_BUFFER_STATE scanbufhandle;
-static char *scanbuf;
+static char *scanbuf; // FIXME
%}
+%option reentrant
+%option bison-bridge
%option 8bit
%option never-interactive
%option nodefault
@@ -53,12 +50,12 @@ float ({integer}|{real})([eE]{integer})?
%%
-{range} seg_yylval.text = yytext; return RANGE;
-{plumin} seg_yylval.text = yytext; return PLUMIN;
-{float} seg_yylval.text = yytext; return SEGFLOAT;
-\< seg_yylval.text = "<"; return EXTENSION;
-\> seg_yylval.text = ">"; return EXTENSION;
-\~ seg_yylval.text = "~"; return EXTENSION;
+{range} yylval->text = yytext; return RANGE;
+{plumin} yylval->text = yytext; return PLUMIN;
+{float} yylval->text = yytext; return SEGFLOAT;
+\< yylval->text = "<"; return EXTENSION;
+\> yylval->text = ">"; return EXTENSION;
+\~ yylval->text = "~"; return EXTENSION;
[ \t\n\r\f\v]+ /* discard spaces */
. return yytext[0]; /* alert parser of the garbage */
@@ -67,8 +64,10 @@ float ({integer}|{real})([eE]{integer})?
/* LCOV_EXCL_STOP */
void
-seg_yyerror(SEG *result, struct Node *escontext, const char *message)
+seg_yyerror(SEG *result, struct Node *escontext, yyscan_t yyscanner, const
char *message)
{
+ struct yyguts_t * yyg = (struct yyguts_t *) yyscanner; /* needed for
yytext macro */
+
/* if we already reported an error, don't overwrite it */
if (SOFT_ERROR_OCCURRED(escontext))
return;
@@ -96,15 +95,15 @@ seg_yyerror(SEG *result, struct Node *escontext, const char
*message)
* Called before any actual parsing is done
*/
void
-seg_scanner_init(const char *str)
+seg_scanner_init(const char *str, yyscan_t *yyscannerp)
{
Size slen = strlen(str);
+ yyscan_t yyscanner;
- /*
- * Might be left over after ereport()
- */
- if (YY_CURRENT_BUFFER)
- yy_delete_buffer(YY_CURRENT_BUFFER);
+ if (yylex_init(yyscannerp) != 0)
+ elog(ERROR, "yylex_init() failed: %m");
+
+ yyscanner = *yyscannerp;
/*
* Make a scan buffer with special termination needed by flex.
@@ -112,9 +111,7 @@ seg_scanner_init(const char *str)
scanbuf = palloc(slen + 2);
memcpy(scanbuf, str, slen);
scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
- scanbufhandle = yy_scan_buffer(scanbuf, slen + 2);
-
- BEGIN(INITIAL);
+ yy_scan_buffer(scanbuf, slen + 2, yyscanner);
}
@@ -122,8 +119,8 @@ seg_scanner_init(const char *str)
* Called after parsing is done to clean up after seg_scanner_init()
*/
void
-seg_scanner_finish(void)
+seg_scanner_finish(yyscan_t yyscanner)
{
- yy_delete_buffer(scanbufhandle);
+ yylex_destroy(yyscanner);
pfree(scanbuf);
}
--
2.47.1
From d203f22ea4523c5826556fcfd854db196d9e14dd Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Mon, 2 Dec 2024 10:35:37 +0100
Subject: [PATCH v0 05/15] seg: Use palloc() instead of malloc() for flex
---
contrib/seg/segscan.l | 30 ++++++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/contrib/seg/segscan.l b/contrib/seg/segscan.l
index 35c3a11ac14..004ac07fae2 100644
--- a/contrib/seg/segscan.l
+++ b/contrib/seg/segscan.l
@@ -38,6 +38,9 @@ static char *scanbuf; // FIXME
%option noinput
%option nounput
%option noyywrap
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
%option warn
%option prefix="seg_yy"
@@ -124,3 +127,30 @@ seg_scanner_finish(yyscan_t yyscanner)
yylex_destroy(yyscanner);
pfree(scanbuf);
}
+
+/*
+ * Interface functions to make flex use palloc() instead of malloc().
+ * It'd be better to make these static, but flex insists otherwise.
+ */
+
+void *
+yyalloc(yy_size_t size, yyscan_t yyscanner)
+{
+ return palloc(size);
+}
+
+void *
+yyrealloc(void *ptr, yy_size_t size, yyscan_t yyscanner)
+{
+ if (ptr)
+ return repalloc(ptr, size);
+ else
+ return palloc(size);
+}
+
+void
+yyfree(void *ptr, yyscan_t yyscanner)
+{
+ if (ptr)
+ pfree(ptr);
+}
--
2.47.1
From c93074ba3949a8edbb5917ceefc80fb020822085 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Mon, 2 Dec 2024 10:35:37 +0100
Subject: [PATCH v0 06/15] seg: Simplify flex scan buffer management
---
contrib/seg/segscan.l | 13 +------------
1 file changed, 1 insertion(+), 12 deletions(-)
diff --git a/contrib/seg/segscan.l b/contrib/seg/segscan.l
index 004ac07fae2..2b35cd4e67a 100644
--- a/contrib/seg/segscan.l
+++ b/contrib/seg/segscan.l
@@ -25,9 +25,6 @@ fprintf_to_ereport(const char *fmt, const char *msg)
{
ereport(ERROR, (errmsg_internal("%s", msg)));
}
-
-/* Handles to the buffer that the lexer uses internally */
-static char *scanbuf; // FIXME
%}
%option reentrant
@@ -100,7 +97,6 @@ seg_yyerror(SEG *result, struct Node *escontext, yyscan_t
yyscanner, const char
void
seg_scanner_init(const char *str, yyscan_t *yyscannerp)
{
- Size slen = strlen(str);
yyscan_t yyscanner;
if (yylex_init(yyscannerp) != 0)
@@ -108,13 +104,7 @@ seg_scanner_init(const char *str, yyscan_t *yyscannerp)
yyscanner = *yyscannerp;
- /*
- * Make a scan buffer with special termination needed by flex.
- */
- scanbuf = palloc(slen + 2);
- memcpy(scanbuf, str, slen);
- scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
- yy_scan_buffer(scanbuf, slen + 2, yyscanner);
+ yy_scan_string(str, yyscanner);
}
@@ -125,7 +115,6 @@ void
seg_scanner_finish(yyscan_t yyscanner)
{
yylex_destroy(yyscanner);
- pfree(scanbuf);
}
/*
--
2.47.1
From 7e1af47bdec122c1ec939912943c968ec3862d73 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Mon, 2 Dec 2024 10:35:37 +0100
Subject: [PATCH v0 07/15] replication parser: pure parser and reentrant
scanner
---
src/backend/replication/repl_gram.y | 9 ++--
src/backend/replication/repl_scanner.l | 54 ++++++++++-----------
src/backend/replication/walsender.c | 11 +++--
src/include/replication/walsender_private.h | 18 ++++---
4 files changed, 47 insertions(+), 45 deletions(-)
diff --git a/src/backend/replication/repl_gram.y
b/src/backend/replication/repl_gram.y
index 06daa954813..4fa71377e20 100644
--- a/src/backend/replication/repl_gram.y
+++ b/src/backend/replication/repl_gram.y
@@ -24,10 +24,6 @@
#include "repl_gram.h"
-/* silence -Wmissing-variable-declarations */
-extern int replication_yychar;
-extern int replication_yynerrs;
-
/* Result of the parsing is returned here */
Node *replication_parse_result;
@@ -43,6 +39,9 @@ Node *replication_parse_result;
%}
+%parse-param {yyscan_t yyscanner}
+%lex-param {yyscan_t yyscanner}
+%pure-parser
%expect 0
%name-prefix="replication_yy"
@@ -106,6 +105,8 @@ Node *replication_parse_result;
firstcmd: command opt_semicolon
{
replication_parse_result = $1;
+
+ (void) yynerrs; /* suppress compiler
warning */
}
;
diff --git a/src/backend/replication/repl_scanner.l
b/src/backend/replication/repl_scanner.l
index e7def800655..99f2efbb8f9 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -38,14 +38,11 @@ fprintf_to_ereport(const char *fmt, const char *msg)
ereport(ERROR, (errmsg_internal("%s", msg)));
}
-/* Handle to the buffer that the lexer uses internally */
-static YY_BUFFER_STATE scanbufhandle;
-
/* Pushed-back token (we only handle one) */
-static int repl_pushed_back_token;
+static int repl_pushed_back_token; /* FIXME */
/* Work area for collecting literals */
-static StringInfoData litbuf;
+static StringInfoData litbuf; /* FIXME */
static void startlit(void);
static char *litbufdup(void);
@@ -56,6 +53,8 @@ static void addlitchar(unsigned char ychar);
%}
+%option reentrant
+%option bison-bridge
%option 8bit
%option never-interactive
%option nodefault
@@ -142,7 +141,7 @@ UPLOAD_MANIFEST { return K_UPLOAD_MANIFEST; }
{space}+ { /* do nothing */ }
{digit}+ {
- replication_yylval.uintval =
strtoul(yytext, NULL, 10);
+ yylval->uintval = strtoul(yytext, NULL,
10);
return UCONST;
}
@@ -150,8 +149,8 @@ UPLOAD_MANIFEST { return K_UPLOAD_MANIFEST; }
uint32 hi,
lo;
if (sscanf(yytext, "%X/%X", &hi, &lo)
!= 2)
- replication_yyerror("invalid
streaming start location");
- replication_yylval.recptr = ((uint64)
hi) << 32 | lo;
+ replication_yyerror("invalid
streaming start location", yyscanner);
+ yylval->recptr = ((uint64) hi) << 32 |
lo;
return RECPTR;
}
@@ -163,7 +162,7 @@ UPLOAD_MANIFEST { return K_UPLOAD_MANIFEST; }
<xq>{quotestop} {
yyless(1);
BEGIN(INITIAL);
- replication_yylval.str = litbufdup();
+ yylval->str = litbufdup();
return SCONST;
}
@@ -185,9 +184,9 @@ UPLOAD_MANIFEST { return K_UPLOAD_MANIFEST; }
yyless(1);
BEGIN(INITIAL);
- replication_yylval.str = litbufdup();
- len = strlen(replication_yylval.str);
-
truncate_identifier(replication_yylval.str, len, true);
+ yylval->str = litbufdup();
+ len = strlen(yylval->str);
+ truncate_identifier(yylval->str, len,
true);
return IDENT;
}
@@ -198,7 +197,7 @@ UPLOAD_MANIFEST { return K_UPLOAD_MANIFEST; }
{identifier} {
int len =
strlen(yytext);
- replication_yylval.str =
downcase_truncate_identifier(yytext, len, true);
+ yylval->str =
downcase_truncate_identifier(yytext, len, true);
return IDENT;
}
@@ -207,7 +206,7 @@ UPLOAD_MANIFEST { return K_UPLOAD_MANIFEST; }
return yytext[0];
}
-<xq,xd><<EOF>> { replication_yyerror("unterminated quoted string"); }
+<xq,xd><<EOF>> { replication_yyerror("unterminated quoted string", yyscanner);
}
<<EOF>> {
@@ -243,7 +242,7 @@ addlitchar(unsigned char ychar)
}
void
-replication_yyerror(const char *message)
+replication_yyerror(yyscan_t yyscanner, const char *message)
{
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
@@ -252,16 +251,16 @@ replication_yyerror(const char *message)
void
-replication_scanner_init(const char *str)
+replication_scanner_init(const char *str, yyscan_t *yyscannerp)
{
Size slen = strlen(str);
char *scanbuf;
+ yyscan_t yyscanner;
- /*
- * Might be left over after ereport()
- */
- if (YY_CURRENT_BUFFER)
- yy_delete_buffer(YY_CURRENT_BUFFER);
+ if (yylex_init(yyscannerp) != 0)
+ elog(ERROR, "yylex_init() failed: %m");
+
+ yyscanner = *yyscannerp;
/*
* Make a scan buffer with special termination needed by flex.
@@ -269,18 +268,16 @@ replication_scanner_init(const char *str)
scanbuf = (char *) palloc(slen + 2);
memcpy(scanbuf, str, slen);
scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
- scanbufhandle = yy_scan_buffer(scanbuf, slen + 2);
+ yy_scan_buffer(scanbuf, slen + 2, yyscanner);
/* Make sure we start in proper state */
- BEGIN(INITIAL);
repl_pushed_back_token = 0;
}
void
-replication_scanner_finish(void)
+replication_scanner_finish(yyscan_t yyscanner)
{
- yy_delete_buffer(scanbufhandle);
- scanbufhandle = NULL;
+ yylex_destroy(yyscanner);
}
/*
@@ -292,9 +289,10 @@ replication_scanner_finish(void)
* IDENT token here, although some other cases are possible.
*/
bool
-replication_scanner_is_replication_command(void)
+replication_scanner_is_replication_command(yyscan_t yyscanner)
{
- int first_token = replication_yylex();
+ YYSTYPE dummy;
+ int first_token = replication_yylex(&dummy,
yyscanner);
switch (first_token)
{
diff --git a/src/backend/replication/walsender.c
b/src/backend/replication/walsender.c
index 371eef3dddc..4971bcfb765 100644
--- a/src/backend/replication/walsender.c
+++ b/src/backend/replication/walsender.c
@@ -1951,6 +1951,7 @@ WalSndWaitForWal(XLogRecPtr loc)
bool
exec_replication_command(const char *cmd_string)
{
+ yyscan_t scanner;
int parse_rc;
Node *cmd_node;
const char *cmdtag;
@@ -1990,15 +1991,15 @@ exec_replication_command(const char *cmd_string)
ALLOCSET_DEFAULT_SIZES);
old_context = MemoryContextSwitchTo(cmd_context);
- replication_scanner_init(cmd_string);
+ replication_scanner_init(cmd_string, &scanner);
/*
* Is it a WalSender command?
*/
- if (!replication_scanner_is_replication_command())
+ if (!replication_scanner_is_replication_command(scanner))
{
/* Nope; clean up and get out. */
- replication_scanner_finish();
+ replication_scanner_finish(scanner);
MemoryContextSwitchTo(old_context);
MemoryContextDelete(cmd_context);
@@ -2016,13 +2017,13 @@ exec_replication_command(const char *cmd_string)
/*
* Looks like a WalSender command, so parse it.
*/
- parse_rc = replication_yyparse();
+ parse_rc = replication_yyparse(scanner);
if (parse_rc != 0)
ereport(ERROR,
(errcode(ERRCODE_SYNTAX_ERROR),
errmsg_internal("replication command parser
returned %d",
parse_rc)));
- replication_scanner_finish();
+ replication_scanner_finish(scanner);
cmd_node = replication_parse_result;
diff --git a/src/include/replication/walsender_private.h
b/src/include/replication/walsender_private.h
index 41ac736b953..a82ce33c083 100644
--- a/src/include/replication/walsender_private.h
+++ b/src/include/replication/walsender_private.h
@@ -125,13 +125,15 @@ extern void WalSndSetState(WalSndState state);
* Internal functions for parsing the replication grammar, in repl_gram.y and
* repl_scanner.l
*/
-extern int replication_yyparse(void);
-extern int replication_yylex(void);
-extern void replication_yyerror(const char *message) pg_attribute_noreturn();
-extern void replication_scanner_init(const char *str);
-extern void replication_scanner_finish(void);
-extern bool replication_scanner_is_replication_command(void);
-
-extern PGDLLIMPORT Node *replication_parse_result;
+union YYSTYPE;
+typedef void *yyscan_t;
+extern int replication_yyparse(yyscan_t yyscanner);
+extern int replication_yylex(union YYSTYPE *yylval_param, yyscan_t
yyscanner);
+extern void replication_yyerror(yyscan_t yyscanner, const char *message)
pg_attribute_noreturn();
+extern void replication_scanner_init(const char *str, yyscan_t *yyscannerp);
+extern void replication_scanner_finish(yyscan_t yyscanner);
+extern bool replication_scanner_is_replication_command(yyscan_t yyscanner);
+
+extern PGDLLIMPORT Node *replication_parse_result; /* FIXME */
#endif /* _WALSENDER_PRIVATE_H
*/
--
2.47.1
From dd77c80158720cae6df33131014249afa9d11edc Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Mon, 2 Dec 2024 10:35:37 +0100
Subject: [PATCH v0 08/15] replication parser: Use palloc() instead of malloc()
for flex
---
src/backend/replication/repl_scanner.l | 30 ++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/src/backend/replication/repl_scanner.l
b/src/backend/replication/repl_scanner.l
index 99f2efbb8f9..6bd25e77880 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -61,6 +61,9 @@ static void addlitchar(unsigned char ychar);
%option noinput
%option nounput
%option noyywrap
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
%option warn
%option prefix="replication_yy"
@@ -314,3 +317,30 @@ replication_scanner_is_replication_command(yyscan_t
yyscanner)
return false;
}
}
+
+/*
+ * Interface functions to make flex use palloc() instead of malloc().
+ * It'd be better to make these static, but flex insists otherwise.
+ */
+
+void *
+yyalloc(yy_size_t size, yyscan_t yyscanner)
+{
+ return palloc(size);
+}
+
+void *
+yyrealloc(void *ptr, yy_size_t size, yyscan_t yyscanner)
+{
+ if (ptr)
+ return repalloc(ptr, size);
+ else
+ return palloc(size);
+}
+
+void
+yyfree(void *ptr, yyscan_t yyscanner)
+{
+ if (ptr)
+ pfree(ptr);
+}
--
2.47.1
From 2c41a75b06c4e2d3a0bfb70d90d7a6af13c7da92 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Mon, 2 Dec 2024 10:35:37 +0100
Subject: [PATCH v0 09/15] replication parser: Simplify flex scan buffer
management
---
src/backend/replication/repl_scanner.l | 10 +---------
1 file changed, 1 insertion(+), 9 deletions(-)
diff --git a/src/backend/replication/repl_scanner.l
b/src/backend/replication/repl_scanner.l
index 6bd25e77880..56f93e681ed 100644
--- a/src/backend/replication/repl_scanner.l
+++ b/src/backend/replication/repl_scanner.l
@@ -256,8 +256,6 @@ replication_yyerror(yyscan_t yyscanner, const char *message)
void
replication_scanner_init(const char *str, yyscan_t *yyscannerp)
{
- Size slen = strlen(str);
- char *scanbuf;
yyscan_t yyscanner;
if (yylex_init(yyscannerp) != 0)
@@ -265,13 +263,7 @@ replication_scanner_init(const char *str, yyscan_t
*yyscannerp)
yyscanner = *yyscannerp;
- /*
- * Make a scan buffer with special termination needed by flex.
- */
- scanbuf = (char *) palloc(slen + 2);
- memcpy(scanbuf, str, slen);
- scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
- yy_scan_buffer(scanbuf, slen + 2, yyscanner);
+ yy_scan_string(str, yyscanner);
/* Make sure we start in proper state */
repl_pushed_back_token = 0;
--
2.47.1
From 11e17f667d4f90d496c6c1ec99df8b198304492c Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Mon, 2 Dec 2024 10:35:37 +0100
Subject: [PATCH v0 10/15] syncrep parser: pure parser and reentrant scanner
---
src/backend/replication/syncrep.c | 7 ++--
src/backend/replication/syncrep_gram.y | 12 ++++---
src/backend/replication/syncrep_scanner.l | 43 +++++++++++------------
src/include/replication/syncrep.h | 12 ++++---
4 files changed, 38 insertions(+), 36 deletions(-)
diff --git a/src/backend/replication/syncrep.c
b/src/backend/replication/syncrep.c
index e1126734ef5..22a2c7fc409 100644
--- a/src/backend/replication/syncrep.c
+++ b/src/backend/replication/syncrep.c
@@ -992,6 +992,7 @@ check_synchronous_standby_names(char **newval, void
**extra, GucSource source)
{
if (*newval != NULL && (*newval)[0] != '\0')
{
+ yyscan_t scanner;
int parse_rc;
SyncRepConfigData *pconf;
@@ -1000,9 +1001,9 @@ check_synchronous_standby_names(char **newval, void
**extra, GucSource source)
syncrep_parse_error_msg = NULL;
/* Parse the synchronous_standby_names string */
- syncrep_scanner_init(*newval);
- parse_rc = syncrep_yyparse();
- syncrep_scanner_finish();
+ syncrep_scanner_init(*newval, &scanner);
+ parse_rc = syncrep_yyparse(scanner);
+ syncrep_scanner_finish(scanner);
if (parse_rc != 0 || syncrep_parse_result == NULL)
{
diff --git a/src/backend/replication/syncrep_gram.y
b/src/backend/replication/syncrep_gram.y
index e4d9962226c..00b5bf0e522 100644
--- a/src/backend/replication/syncrep_gram.y
+++ b/src/backend/replication/syncrep_gram.y
@@ -26,10 +26,6 @@ char *syncrep_parse_error_msg;
static SyncRepConfigData *create_syncrep_config(const char *num_sync,
List *members, uint8 syncrep_method);
-/* silence -Wmissing-variable-declarations */
-extern int syncrep_yychar;
-extern int syncrep_yynerrs;
-
/*
* Bison doesn't allocate anything that needs to live across parser calls,
* so we can easily have it use palloc instead of malloc. This prevents
@@ -40,6 +36,9 @@ extern int syncrep_yynerrs;
%}
+%parse-param {yyscan_t yyscanner}
+%lex-param {yyscan_t yyscanner}
+%pure-parser
%expect 0
%name-prefix="syncrep_yy"
@@ -60,7 +59,10 @@ extern int syncrep_yynerrs;
%%
result:
- standby_config { syncrep_parse_result
= $1; }
+ standby_config {
+
syncrep_parse_result = $1;
+
(void) yynerrs; /* suppress compiler warning */
+ }
;
standby_config:
diff --git a/src/backend/replication/syncrep_scanner.l
b/src/backend/replication/syncrep_scanner.l
index 6defb90f13b..b1edb780502 100644
--- a/src/backend/replication/syncrep_scanner.l
+++ b/src/backend/replication/syncrep_scanner.l
@@ -37,15 +37,14 @@ fprintf_to_ereport(const char *fmt, const char *msg)
ereport(ERROR, (errmsg_internal("%s", msg)));
}
-/* Handles to the buffer that the lexer uses internally */
-static YY_BUFFER_STATE scanbufhandle;
-
-static StringInfoData xdbuf;
+static StringInfoData xdbuf; /* FIXME */
/* LCOV_EXCL_START */
%}
+%option reentrant
+%option bison-bridge
%option 8bit
%option never-interactive
%option nodefault
@@ -92,28 +91,28 @@ xdinside [^"]+
appendStringInfoString(&xdbuf, yytext);
}
<xd>{xdstop} {
- syncrep_yylval.str = xdbuf.data;
+ yylval->str = xdbuf.data;
xdbuf.data = NULL;
BEGIN(INITIAL);
return NAME;
}
<xd><<EOF>> {
- syncrep_yyerror("unterminated quoted
identifier");
+ syncrep_yyerror("unterminated quoted
identifier", yyscanner);
return JUNK;
}
{identifier} {
- syncrep_yylval.str = pstrdup(yytext);
+ yylval->str = pstrdup(yytext);
return NAME;
}
{digit}+ {
- syncrep_yylval.str = pstrdup(yytext);
+ yylval->str = pstrdup(yytext);
return NUM;
}
"*" {
- syncrep_yylval.str = "*";
+ yylval->str = "*";
return NAME;
}
@@ -128,8 +127,10 @@ xdinside [^"]+
/* Needs to be here for access to yytext */
void
-syncrep_yyerror(const char *message)
+syncrep_yyerror(yyscan_t yyscanner, const char *message)
{
+ struct yyguts_t * yyg = (struct yyguts_t *) yyscanner; /* needed for
yytext macro */
+
/* report only the first error in a parse operation */
if (syncrep_parse_error_msg)
return;
@@ -142,16 +143,16 @@ syncrep_yyerror(const char *message)
}
void
-syncrep_scanner_init(const char *str)
+syncrep_scanner_init(const char *str, yyscan_t *yyscannerp)
{
Size slen = strlen(str);
char *scanbuf;
+ yyscan_t yyscanner;
- /*
- * Might be left over after ereport()
- */
- if (YY_CURRENT_BUFFER)
- yy_delete_buffer(YY_CURRENT_BUFFER);
+ if (yylex_init(yyscannerp) != 0)
+ elog(ERROR, "yylex_init() failed: %m");
+
+ yyscanner = *yyscannerp;
/*
* Make a scan buffer with special termination needed by flex.
@@ -159,15 +160,11 @@ syncrep_scanner_init(const char *str)
scanbuf = (char *) palloc(slen + 2);
memcpy(scanbuf, str, slen);
scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
- scanbufhandle = yy_scan_buffer(scanbuf, slen + 2);
-
- /* Make sure we start in proper state */
- BEGIN(INITIAL);
+ yy_scan_buffer(scanbuf, slen + 2, yyscanner);
}
void
-syncrep_scanner_finish(void)
+syncrep_scanner_finish(yyscan_t yyscanner)
{
- yy_delete_buffer(scanbufhandle);
- scanbufhandle = NULL;
+ yylex_destroy(yyscanner);
}
diff --git a/src/include/replication/syncrep.h
b/src/include/replication/syncrep.h
index ea439e6da60..b4134dbc91d 100644
--- a/src/include/replication/syncrep.h
+++ b/src/include/replication/syncrep.h
@@ -100,10 +100,12 @@ extern void SyncRepUpdateSyncStandbysDefined(void);
* Internal functions for parsing synchronous_standby_names grammar,
* in syncrep_gram.y and syncrep_scanner.l
*/
-extern int syncrep_yyparse(void);
-extern int syncrep_yylex(void);
-extern void syncrep_yyerror(const char *str);
-extern void syncrep_scanner_init(const char *str);
-extern void syncrep_scanner_finish(void);
+union YYSTYPE;
+typedef void *yyscan_t;
+extern int syncrep_yyparse(yyscan_t yyscanner);
+extern int syncrep_yylex(union YYSTYPE *yylval_param, yyscan_t yyscanner);
+extern void syncrep_yyerror(yyscan_t yyscanner, const char *str);
+extern void syncrep_scanner_init(const char *str, yyscan_t *yyscannerp);
+extern void syncrep_scanner_finish(yyscan_t yyscanner);
#endif /* _SYNCREP_H */
--
2.47.1
From 142dfbb6a92cd970709a24462a538191030462ae Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Mon, 2 Dec 2024 10:35:37 +0100
Subject: [PATCH v0 11/15] syncrep parser: Use palloc() instead of malloc() for
flex
---
src/backend/replication/syncrep_scanner.l | 30 +++++++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/src/backend/replication/syncrep_scanner.l
b/src/backend/replication/syncrep_scanner.l
index b1edb780502..ac2eecd7804 100644
--- a/src/backend/replication/syncrep_scanner.l
+++ b/src/backend/replication/syncrep_scanner.l
@@ -51,6 +51,9 @@ static StringInfoData xdbuf; /* FIXME */
%option noinput
%option nounput
%option noyywrap
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
%option warn
%option prefix="syncrep_yy"
@@ -168,3 +171,30 @@ syncrep_scanner_finish(yyscan_t yyscanner)
{
yylex_destroy(yyscanner);
}
+
+/*
+ * Interface functions to make flex use palloc() instead of malloc().
+ * It'd be better to make these static, but flex insists otherwise.
+ */
+
+void *
+yyalloc(yy_size_t size, yyscan_t yyscanner)
+{
+ return palloc(size);
+}
+
+void *
+yyrealloc(void *ptr, yy_size_t size, yyscan_t yyscanner)
+{
+ if (ptr)
+ return repalloc(ptr, size);
+ else
+ return palloc(size);
+}
+
+void
+yyfree(void *ptr, yyscan_t yyscanner)
+{
+ if (ptr)
+ pfree(ptr);
+}
--
2.47.1
From 88ca33889a7bc2f6986d0fbb8c8fc3c201d43ab6 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Mon, 2 Dec 2024 10:35:37 +0100
Subject: [PATCH v0 12/15] syncrep parser: Simplify flex scan buffer management
---
src/backend/replication/syncrep_scanner.l | 10 +---------
1 file changed, 1 insertion(+), 9 deletions(-)
diff --git a/src/backend/replication/syncrep_scanner.l
b/src/backend/replication/syncrep_scanner.l
index ac2eecd7804..9c0f4fc2c19 100644
--- a/src/backend/replication/syncrep_scanner.l
+++ b/src/backend/replication/syncrep_scanner.l
@@ -148,8 +148,6 @@ syncrep_yyerror(yyscan_t yyscanner, const char *message)
void
syncrep_scanner_init(const char *str, yyscan_t *yyscannerp)
{
- Size slen = strlen(str);
- char *scanbuf;
yyscan_t yyscanner;
if (yylex_init(yyscannerp) != 0)
@@ -157,13 +155,7 @@ syncrep_scanner_init(const char *str, yyscan_t *yyscannerp)
yyscanner = *yyscannerp;
- /*
- * Make a scan buffer with special termination needed by flex.
- */
- scanbuf = (char *) palloc(slen + 2);
- memcpy(scanbuf, str, slen);
- scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
- yy_scan_buffer(scanbuf, slen + 2, yyscanner);
+ yy_scan_string(str, yyscanner);
}
void
--
2.47.1
From 0b5c772dd5ce8bf9a32e5072adc64798ca63b0ae Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Mon, 2 Dec 2024 10:35:37 +0100
Subject: [PATCH v0 13/15] syncrep parser: Use flex yyextra
Use flex yyextra to handle context information, instead of global
variables. This complements the earlier patch to make the scanner
reentrant.
---
src/backend/replication/syncrep_scanner.l | 19 +++++++++++++------
1 file changed, 13 insertions(+), 6 deletions(-)
diff --git a/src/backend/replication/syncrep_scanner.l
b/src/backend/replication/syncrep_scanner.l
index 9c0f4fc2c19..85ae4e60604 100644
--- a/src/backend/replication/syncrep_scanner.l
+++ b/src/backend/replication/syncrep_scanner.l
@@ -37,7 +37,11 @@ fprintf_to_ereport(const char *fmt, const char *msg)
ereport(ERROR, (errmsg_internal("%s", msg)));
}
-static StringInfoData xdbuf; /* FIXME */
+struct syncrep_yy_extra_type
+{
+ StringInfoData xdbuf;
+};
+#define YY_EXTRA_TYPE struct syncrep_yy_extra_type *
/* LCOV_EXCL_START */
@@ -84,18 +88,18 @@ xdinside [^"]+
[Ff][Ii][Rr][Ss][Tt] { return FIRST; }
{xdstart} {
- initStringInfo(&xdbuf);
+ initStringInfo(&yyextra->xdbuf);
BEGIN(xd);
}
<xd>{xddouble} {
- appendStringInfoChar(&xdbuf, '"');
+ appendStringInfoChar(&yyextra->xdbuf, '"');
}
<xd>{xdinside} {
- appendStringInfoString(&xdbuf, yytext);
+ appendStringInfoString(&yyextra->xdbuf, yytext);
}
<xd>{xdstop} {
- yylval->str = xdbuf.data;
- xdbuf.data = NULL;
+ yylval->str = yyextra->xdbuf.data;
+ yyextra->xdbuf.data = NULL;
BEGIN(INITIAL);
return NAME;
}
@@ -149,12 +153,15 @@ void
syncrep_scanner_init(const char *str, yyscan_t *yyscannerp)
{
yyscan_t yyscanner;
+ struct syncrep_yy_extra_type yyext;
if (yylex_init(yyscannerp) != 0)
elog(ERROR, "yylex_init() failed: %m");
yyscanner = *yyscannerp;
+ yyset_extra(&yyext, yyscanner);
+
yy_scan_string(str, yyscanner);
}
--
2.47.1
From 1441e2d43496a38c722e01e5dea3ac9152fbdde9 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Mon, 2 Dec 2024 10:35:37 +0100
Subject: [PATCH v0 14/15] jsonpath scanner: reentrant scanner
Note: The parser was already pure.
---
src/backend/utils/adt/jsonpath_gram.y | 2 +
src/backend/utils/adt/jsonpath_internal.h | 9 +-
src/backend/utils/adt/jsonpath_scan.l | 114 ++++++++--------------
3 files changed, 50 insertions(+), 75 deletions(-)
diff --git a/src/backend/utils/adt/jsonpath_gram.y
b/src/backend/utils/adt/jsonpath_gram.y
index 8733a0eac66..de5a455c96d 100644
--- a/src/backend/utils/adt/jsonpath_gram.y
+++ b/src/backend/utils/adt/jsonpath_gram.y
@@ -60,8 +60,10 @@ static bool makeItemLikeRegex(JsonPathParseItem *expr,
%name-prefix="jsonpath_yy"
%parse-param {JsonPathParseResult **result}
%parse-param {struct Node *escontext}
+%parse-param {yyscan_t yyscanner}
%lex-param {JsonPathParseResult **result}
%lex-param {struct Node *escontext}
+%lex-param {yyscan_t yyscanner}
%union
{
diff --git a/src/backend/utils/adt/jsonpath_internal.h
b/src/backend/utils/adt/jsonpath_internal.h
index 6cd6d8b652d..71f885475dd 100644
--- a/src/backend/utils/adt/jsonpath_internal.h
+++ b/src/backend/utils/adt/jsonpath_internal.h
@@ -22,17 +22,22 @@ typedef struct JsonPathString
int total;
} JsonPathString;
+typedef void *yyscan_t;
+
#include "utils/jsonpath.h"
#include "jsonpath_gram.h"
#define YY_DECL extern int jsonpath_yylex(YYSTYPE *yylval_param, \
JsonPathParseResult
**result, \
- struct Node
*escontext)
+ struct Node
*escontext, \
+ yyscan_t yyscanner)
YY_DECL;
extern int jsonpath_yyparse(JsonPathParseResult **result,
- struct Node
*escontext);
+ struct Node *escontext,
+ yyscan_t yyscanner);
extern void jsonpath_yyerror(JsonPathParseResult **result,
struct Node *escontext,
+ yyscan_t yyscanner,
const char *message);
#endif /* JSONPATH_INTERNAL_H
*/
diff --git a/src/backend/utils/adt/jsonpath_scan.l
b/src/backend/utils/adt/jsonpath_scan.l
index f5a85de36f5..700c17712d0 100644
--- a/src/backend/utils/adt/jsonpath_scan.l
+++ b/src/backend/utils/adt/jsonpath_scan.l
@@ -30,18 +30,13 @@
}
%{
-static JsonPathString scanstring;
-
-/* Handles to the buffer that the lexer uses internally */
-static YY_BUFFER_STATE scanbufhandle;
-static char *scanbuf;
-static int scanbuflen;
+static JsonPathString scanstring; /* FIXME */
static void addstring(bool init, char *s, int l);
static void addchar(bool init, char c);
static enum yytokentype checkKeyword(void);
-static bool parseUnicode(char *s, int l, struct Node *escontext);
-static bool parseHexChar(char *s, struct Node *escontext);
+static bool parseUnicode(char *s, int l, struct Node *escontext, yyscan_t
yyscanner);
+static bool parseHexChar(char *s, struct Node *escontext, yyscan_t yyscanner);
/* Avoid exit() on fatal scanner errors (a bit ugly -- see yy_fatal_error) */
#undef fprintf
@@ -65,6 +60,7 @@ fprintf_to_ereport(const char *fmt, const char *msg)
%option noyywrap
%option warn
%option prefix="jsonpath_yy"
+%option reentrant
%option bison-bridge
%option noyyalloc
%option noyyrealloc
@@ -160,23 +156,23 @@ hex_fail \\x{hexdigit}{0,1}
<xnq,xq,xvq>\\v { addchar(false, '\v'); }
<xnq,xq,xvq>{unicode}+ {
- if
(!parseUnicode(yytext, yyleng, escontext))
+ if
(!parseUnicode(yytext, yyleng, escontext, yyscanner))
yyterminate();
}
<xnq,xq,xvq>{hex_char} {
- if
(!parseHexChar(yytext, escontext))
+ if
(!parseHexChar(yytext, escontext, yyscanner))
yyterminate();
}
<xnq,xq,xvq>{unicode}*{unicodefail} {
-
jsonpath_yyerror(NULL, escontext,
+
jsonpath_yyerror(NULL, escontext, yyscanner,
"invalid Unicode escape sequence");
yyterminate();
}
<xnq,xq,xvq>{hex_fail} {
-
jsonpath_yyerror(NULL, escontext,
+
jsonpath_yyerror(NULL, escontext, yyscanner,
"invalid hexadecimal character sequence");
yyterminate();
}
@@ -184,20 +180,20 @@ hex_fail \\x{hexdigit}{0,1}
<xnq,xq,xvq>{unicode}+\\ {
/* throw back
the \\, and treat as unicode */
yyless(yyleng -
1);
- if
(!parseUnicode(yytext, yyleng, escontext))
+ if
(!parseUnicode(yytext, yyleng, escontext, yyscanner))
yyterminate();
}
<xnq,xq,xvq>\\. { addchar(false, yytext[1]); }
<xnq,xq,xvq>\\ {
-
jsonpath_yyerror(NULL, escontext,
+
jsonpath_yyerror(NULL, escontext, yyscanner,
"unexpected end after backslash");
yyterminate();
}
<xq,xvq><<EOF>> {
-
jsonpath_yyerror(NULL, escontext,
+
jsonpath_yyerror(NULL, escontext, yyscanner,
"unterminated quoted string");
yyterminate();
}
@@ -223,7 +219,7 @@ hex_fail \\x{hexdigit}{0,1}
<xc>\* { }
<xc><<EOF>> {
-
jsonpath_yyerror(NULL, escontext,
+
jsonpath_yyerror(NULL, escontext, yyscanner,
"unexpected end of comment");
yyterminate();
}
@@ -313,22 +309,22 @@ hex_fail \\x{hexdigit}{0,1}
}
{realfail} {
-
jsonpath_yyerror(NULL, escontext,
+
jsonpath_yyerror(NULL, escontext, yyscanner,
"invalid numeric literal");
yyterminate();
}
{decinteger_junk} {
-
jsonpath_yyerror(NULL, escontext,
+
jsonpath_yyerror(NULL, escontext, yyscanner,
"trailing junk after numeric literal");
yyterminate();
}
{decimal_junk} {
-
jsonpath_yyerror(NULL, escontext,
+
jsonpath_yyerror(NULL, escontext, yyscanner,
"trailing junk after numeric literal");
yyterminate();
}
{real_junk} {
-
jsonpath_yyerror(NULL, escontext,
+
jsonpath_yyerror(NULL, escontext, yyscanner,
"trailing junk after numeric literal");
yyterminate();
}
@@ -356,8 +352,11 @@ hex_fail \\x{hexdigit}{0,1}
void
jsonpath_yyerror(JsonPathParseResult **result, struct Node *escontext,
+ yyscan_t yyscanner,
const char *message)
{
+ struct yyguts_t * yyg = (struct yyguts_t *) yyscanner; /* needed for
yytext macro */
+
/* don't overwrite escontext if it's already been set */
if (SOFT_ERROR_OCCURRED(escontext))
return;
@@ -470,44 +469,6 @@ checkKeyword()
return res;
}
-/*
- * Called before any actual parsing is done
- */
-static void
-jsonpath_scanner_init(const char *str, int slen)
-{
- if (slen <= 0)
- slen = strlen(str);
-
- /*
- * Might be left over after ereport()
- */
- yy_init_globals();
-
- /*
- * Make a scan buffer with special termination needed by flex.
- */
-
- scanbuflen = slen;
- scanbuf = palloc(slen + 2);
- memcpy(scanbuf, str, slen);
- scanbuf[slen] = scanbuf[slen + 1] = YY_END_OF_BUFFER_CHAR;
- scanbufhandle = yy_scan_buffer(scanbuf, slen + 2);
-
- BEGIN(INITIAL);
-}
-
-
-/*
- * Called after parsing is done to clean up after jsonpath_scanner_init()
- */
-static void
-jsonpath_scanner_finish(void)
-{
- yy_delete_buffer(scanbufhandle);
- pfree(scanbuf);
-}
-
/*
* Resize scanstring so that it can append string of given length.
* Reinitialize if required.
@@ -556,20 +517,27 @@ JsonPathParseResult *
parsejsonpath(const char *str, int len, struct Node *escontext)
{
JsonPathParseResult *parseresult;
+ yyscan_t scanner;
+
+ if (jsonpath_yylex_init(&scanner) != 0)
+ elog(ERROR, "yylex_init() failed: %m");
+
+ if (len <= 0)
+ len = strlen(str);
- jsonpath_scanner_init(str, len);
+ jsonpath_yy_scan_bytes(str, len, scanner);
- if (jsonpath_yyparse(&parseresult, escontext) != 0)
- jsonpath_yyerror(NULL, escontext, "invalid input"); /*
shouldn't happen */
+ if (jsonpath_yyparse(&parseresult, escontext, scanner) != 0)
+ jsonpath_yyerror(NULL, escontext, scanner, "invalid input"); /*
shouldn't happen */
- jsonpath_scanner_finish();
+ jsonpath_yylex_destroy(scanner);
return parseresult;
}
/* Turn hex character into integer */
static bool
-hexval(char c, int *result, struct Node *escontext)
+hexval(char c, int *result, struct Node *escontext, yyscan_t yyscanner)
{
if (c >= '0' && c <= '9')
{
@@ -586,7 +554,7 @@ hexval(char c, int *result, struct Node *escontext)
*result = c - 'A' + 0xA;
return true;
}
- jsonpath_yyerror(NULL, escontext, "invalid hexadecimal digit");
+ jsonpath_yyerror(NULL, escontext, yyscanner, "invalid hexadecimal
digit");
return false;
}
@@ -666,7 +634,7 @@ addUnicode(int ch, int *hi_surrogate, struct Node
*escontext)
* src/backend/utils/adt/json.c
*/
static bool
-parseUnicode(char *s, int l, struct Node *escontext)
+parseUnicode(char *s, int l, struct Node *escontext, yyscan_t yyscanner)
{
int i = 2;
int hi_surrogate = -1;
@@ -680,7 +648,7 @@ parseUnicode(char *s, int l, struct Node *escontext)
{
while (s[++i] != '}' && i < l)
{
- if (!hexval(s[i], &si, escontext))
+ if (!hexval(s[i], &si, escontext, yyscanner))
return false;
ch = (ch << 4) | si;
}
@@ -690,7 +658,7 @@ parseUnicode(char *s, int l, struct Node *escontext)
{
for (j = 0; j < 4 && i < l; j++)
{
- if (!hexval(s[i++], &si, escontext))
+ if (!hexval(s[i++], &si, escontext, yyscanner))
return false;
ch = (ch << 4) | si;
}
@@ -714,12 +682,12 @@ parseUnicode(char *s, int l, struct Node *escontext)
/* Parse sequence of hex-encoded characters */
static bool
-parseHexChar(char *s, struct Node *escontext)
+parseHexChar(char *s, struct Node *escontext, yyscan_t yyscanner)
{
int s2, s3, ch;
- if (!hexval(s[2], &s2, escontext))
+ if (!hexval(s[2], &s2, escontext, yyscanner))
return false;
- if (!hexval(s[3], &s3, escontext))
+ if (!hexval(s[3], &s3, escontext, yyscanner))
return false;
ch = (s2 << 4) | s3;
@@ -733,13 +701,13 @@ parseHexChar(char *s, struct Node *escontext)
*/
void *
-jsonpath_yyalloc(yy_size_t bytes)
+jsonpath_yyalloc(yy_size_t bytes, yyscan_t yyscanner)
{
return palloc(bytes);
}
void *
-jsonpath_yyrealloc(void *ptr, yy_size_t bytes)
+jsonpath_yyrealloc(void *ptr, yy_size_t bytes, yyscan_t yyscanner)
{
if (ptr)
return repalloc(ptr, bytes);
@@ -748,7 +716,7 @@ jsonpath_yyrealloc(void *ptr, yy_size_t bytes)
}
void
-jsonpath_yyfree(void *ptr)
+jsonpath_yyfree(void *ptr, yyscan_t yyscanner)
{
if (ptr)
pfree(ptr);
--
2.47.1
From 806f4173418991462bb2f9ad029c219ab8abe3d9 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Mon, 2 Dec 2024 10:35:37 +0100
Subject: [PATCH v0 15/15] jsonpath scanner: Use flex yyextra
---
src/backend/utils/adt/jsonpath_scan.l | 171 ++++++++++++++------------
1 file changed, 92 insertions(+), 79 deletions(-)
diff --git a/src/backend/utils/adt/jsonpath_scan.l
b/src/backend/utils/adt/jsonpath_scan.l
index 700c17712d0..8ed6c7ddf63 100644
--- a/src/backend/utils/adt/jsonpath_scan.l
+++ b/src/backend/utils/adt/jsonpath_scan.l
@@ -30,11 +30,15 @@
}
%{
-static JsonPathString scanstring; /* FIXME */
+struct jsonpath_yy_extra_type
+{
+ JsonPathString scanstring;
+};
+#define YY_EXTRA_TYPE struct jsonpath_yy_extra_type *
-static void addstring(bool init, char *s, int l);
-static void addchar(bool init, char c);
-static enum yytokentype checkKeyword(void);
+static void addstring(bool init, char *s, int l, yyscan_t yyscanner);
+static void addchar(bool init, char c, yyscan_t yyscanner);
+static enum yytokentype checkKeyword(yyscan_t yyscanner);
static bool parseUnicode(char *s, int l, struct Node *escontext, yyscan_t
yyscanner);
static bool parseHexChar(char *s, struct Node *escontext, yyscan_t yyscanner);
@@ -116,44 +120,44 @@ hex_fail \\x{hexdigit}{0,1}
%%
<xnq>{other}+ {
-
addstring(false, yytext, yyleng);
+
addstring(false, yytext, yyleng, yyscanner);
}
<xnq>{blank}+ {
-
yylval->str = scanstring;
+
yylval->str = yyextra->scanstring;
BEGIN
INITIAL;
- return
checkKeyword();
+ return
checkKeyword(yyscanner);
}
<xnq>\/\* {
-
yylval->str = scanstring;
+
yylval->str = yyextra->scanstring;
BEGIN
xc;
}
<xnq>({special}|\") {
-
yylval->str = scanstring;
+
yylval->str = yyextra->scanstring;
yyless(0);
BEGIN
INITIAL;
- return
checkKeyword();
+ return
checkKeyword(yyscanner);
}
<xnq><<EOF>> {
-
yylval->str = scanstring;
+
yylval->str = yyextra->scanstring;
BEGIN
INITIAL;
- return
checkKeyword();
+ return
checkKeyword(yyscanner);
}
-<xnq,xq,xvq>\\b { addchar(false, '\b'); }
+<xnq,xq,xvq>\\b { addchar(false, '\b',
yyscanner); }
-<xnq,xq,xvq>\\f { addchar(false, '\f'); }
+<xnq,xq,xvq>\\f { addchar(false, '\f',
yyscanner); }
-<xnq,xq,xvq>\\n { addchar(false, '\n'); }
+<xnq,xq,xvq>\\n { addchar(false, '\n',
yyscanner); }
-<xnq,xq,xvq>\\r { addchar(false, '\r'); }
+<xnq,xq,xvq>\\r { addchar(false, '\r',
yyscanner); }
-<xnq,xq,xvq>\\t { addchar(false, '\t'); }
+<xnq,xq,xvq>\\t { addchar(false, '\t',
yyscanner); }
-<xnq,xq,xvq>\\v { addchar(false, '\v'); }
+<xnq,xq,xvq>\\v { addchar(false, '\v',
yyscanner); }
<xnq,xq,xvq>{unicode}+ {
if
(!parseUnicode(yytext, yyleng, escontext, yyscanner))
@@ -184,7 +188,7 @@ hex_fail \\x{hexdigit}{0,1}
yyterminate();
}
-<xnq,xq,xvq>\\. { addchar(false, yytext[1]); }
+<xnq,xq,xvq>\\. { addchar(false, yytext[1],
yyscanner); }
<xnq,xq,xvq>\\ {
jsonpath_yyerror(NULL, escontext, yyscanner,
@@ -199,18 +203,18 @@ hex_fail \\x{hexdigit}{0,1}
}
<xq>\" {
-
yylval->str = scanstring;
+
yylval->str = yyextra->scanstring;
BEGIN
INITIAL;
return
STRING_P;
}
<xvq>\" {
-
yylval->str = scanstring;
+
yylval->str = yyextra->scanstring;
BEGIN
INITIAL;
return
VARIABLE_P;
}
-<xq,xvq>[^\\\"]+ { addstring(false, yytext,
yyleng); }
+<xq,xvq>[^\\\"]+ { addstring(false, yytext,
yyleng, yyscanner); }
<xc>\*\/ { BEGIN INITIAL; }
@@ -246,14 +250,14 @@ hex_fail \\x{hexdigit}{0,1}
\> { return
GREATER_P; }
\${other}+ {
-
addstring(true, yytext + 1, yyleng - 1);
-
addchar(false, '\0');
-
yylval->str = scanstring;
+
addstring(true, yytext + 1, yyleng - 1, yyscanner);
+
addchar(false, '\0', yyscanner);
+
yylval->str = yyextra->scanstring;
return
VARIABLE_P;
}
\$\" {
-
addchar(true, '\0');
+
addchar(true, '\0', yyscanner);
BEGIN
xvq;
}
@@ -262,49 +266,49 @@ hex_fail \\x{hexdigit}{0,1}
{blank}+ { /* ignore */ }
\/\* {
-
addchar(true, '\0');
+
addchar(true, '\0', yyscanner);
BEGIN
xc;
}
{real} {
-
addstring(true, yytext, yyleng);
-
addchar(false, '\0');
-
yylval->str = scanstring;
+
addstring(true, yytext, yyleng, yyscanner);
+
addchar(false, '\0', yyscanner);
+
yylval->str = yyextra->scanstring;
return
NUMERIC_P;
}
{decimal} {
-
addstring(true, yytext, yyleng);
-
addchar(false, '\0');
-
yylval->str = scanstring;
+
addstring(true, yytext, yyleng, yyscanner);
+
addchar(false, '\0', yyscanner);
+
yylval->str = yyextra->scanstring;
return
NUMERIC_P;
}
{decinteger} {
-
addstring(true, yytext, yyleng);
-
addchar(false, '\0');
-
yylval->str = scanstring;
+
addstring(true, yytext, yyleng, yyscanner);
+
addchar(false, '\0', yyscanner);
+
yylval->str = yyextra->scanstring;
return
INT_P;
}
{hexinteger} {
-
addstring(true, yytext, yyleng);
-
addchar(false, '\0');
-
yylval->str = scanstring;
+
addstring(true, yytext, yyleng, yyscanner);
+
addchar(false, '\0', yyscanner);
+
yylval->str = yyextra->scanstring;
return
INT_P;
}
{octinteger} {
-
addstring(true, yytext, yyleng);
-
addchar(false, '\0');
-
yylval->str = scanstring;
+
addstring(true, yytext, yyleng, yyscanner);
+
addchar(false, '\0', yyscanner);
+
yylval->str = yyextra->scanstring;
return
INT_P;
}
{bininteger} {
-
addstring(true, yytext, yyleng);
-
addchar(false, '\0');
-
yylval->str = scanstring;
+
addstring(true, yytext, yyleng, yyscanner);
+
addchar(false, '\0', yyscanner);
+
yylval->str = yyextra->scanstring;
return
INT_P;
}
@@ -329,18 +333,18 @@ hex_fail \\x{hexdigit}{0,1}
yyterminate();
}
\" {
-
addchar(true, '\0');
+
addchar(true, '\0', yyscanner);
BEGIN
xq;
}
\\ {
yyless(0);
-
addchar(true, '\0');
+
addchar(true, '\0', yyscanner);
BEGIN
xnq;
}
{other}+ {
-
addstring(true, yytext, yyleng);
+
addstring(true, yytext, yyleng, yyscanner);
BEGIN
xnq;
}
@@ -350,6 +354,10 @@ hex_fail \\x{hexdigit}{0,1}
/* LCOV_EXCL_STOP */
+/* see scan.l */
+#undef yyextra
+#define yyextra (((struct yyguts_t *) yyscanner)->yyextra_r)
+
void
jsonpath_yyerror(JsonPathParseResult **result, struct Node *escontext,
yyscan_t yyscanner,
@@ -426,9 +434,11 @@ static const JsonPathKeyword keywords[] = {
{ 12,false, TIMESTAMP_TZ_P, "timestamp_tz"},
};
-/* Check if current scanstring value is a keyword */
+/*
+ * Check if current scanstring value is a keyword
+ */
static enum yytokentype
-checkKeyword()
+checkKeyword(yyscan_t yyscanner)
{
int res = IDENT_P;
int diff;
@@ -436,18 +446,18 @@ checkKeyword()
*StopHigh = keywords +
lengthof(keywords),
*StopMiddle;
- if (scanstring.len > keywords[lengthof(keywords) - 1].len)
+ if (yyextra->scanstring.len > keywords[lengthof(keywords) - 1].len)
return res;
while (StopLow < StopHigh)
{
StopMiddle = StopLow + ((StopHigh - StopLow) >> 1);
- if (StopMiddle->len == scanstring.len)
- diff = pg_strncasecmp(StopMiddle->keyword,
scanstring.val,
-
scanstring.len);
+ if (StopMiddle->len == yyextra->scanstring.len)
+ diff = pg_strncasecmp(StopMiddle->keyword,
yyextra->scanstring.val,
+
yyextra->scanstring.len);
else
- diff = StopMiddle->len - scanstring.len;
+ diff = StopMiddle->len - yyextra->scanstring.len;
if (diff < 0)
StopLow = StopMiddle + 1;
@@ -456,8 +466,8 @@ checkKeyword()
else
{
if (StopMiddle->lowercase)
- diff = strncmp(StopMiddle->keyword,
scanstring.val,
- scanstring.len);
+ diff = strncmp(StopMiddle->keyword,
yyextra->scanstring.val,
+
yyextra->scanstring.len);
if (diff == 0)
res = StopMiddle->val;
@@ -474,42 +484,42 @@ checkKeyword()
* Reinitialize if required.
*/
static void
-resizeString(bool init, int appendLen)
+resizeString(bool init, int appendLen, yyscan_t yyscanner)
{
if (init)
{
- scanstring.total = Max(32, appendLen);
- scanstring.val = (char *) palloc(scanstring.total);
- scanstring.len = 0;
+ yyextra->scanstring.total = Max(32, appendLen);
+ yyextra->scanstring.val = (char *)
palloc(yyextra->scanstring.total);
+ yyextra->scanstring.len = 0;
}
else
{
- if (scanstring.len + appendLen >= scanstring.total)
+ if (yyextra->scanstring.len + appendLen >=
yyextra->scanstring.total)
{
- while (scanstring.len + appendLen >= scanstring.total)
- scanstring.total *= 2;
- scanstring.val = repalloc(scanstring.val,
scanstring.total);
+ while (yyextra->scanstring.len + appendLen >=
yyextra->scanstring.total)
+ yyextra->scanstring.total *= 2;
+ yyextra->scanstring.val =
repalloc(yyextra->scanstring.val, yyextra->scanstring.total);
}
}
}
/* Add set of bytes at "s" of length "l" to scanstring */
static void
-addstring(bool init, char *s, int l)
+addstring(bool init, char *s, int l, yyscan_t yyscanner)
{
- resizeString(init, l + 1);
- memcpy(scanstring.val + scanstring.len, s, l);
- scanstring.len += l;
+ resizeString(init, l + 1, yyscanner);
+ memcpy(yyextra->scanstring.val + yyextra->scanstring.len, s, l);
+ yyextra->scanstring.len += l;
}
/* Add single byte "c" to scanstring */
static void
-addchar(bool init, char c)
+addchar(bool init, char c, yyscan_t yyscanner)
{
- resizeString(init, 1);
- scanstring.val[scanstring.len] = c;
+ resizeString(init, 1, yyscanner);
+ yyextra->scanstring.val[yyextra->scanstring.len] = c;
if (c != '\0')
- scanstring.len++;
+ yyextra->scanstring.len++;
}
/* Interface to jsonpath parser */
@@ -518,10 +528,13 @@ parsejsonpath(const char *str, int len, struct Node
*escontext)
{
JsonPathParseResult *parseresult;
yyscan_t scanner;
+ struct jsonpath_yy_extra_type yyext;
if (jsonpath_yylex_init(&scanner) != 0)
elog(ERROR, "yylex_init() failed: %m");
+ yyset_extra(&yyext, scanner);
+
if (len <= 0)
len = strlen(str);
@@ -560,7 +573,7 @@ hexval(char c, int *result, struct Node *escontext,
yyscan_t yyscanner)
/* Add given unicode character to scanstring */
static bool
-addUnicodeChar(int ch, struct Node *escontext)
+addUnicodeChar(int ch, struct Node *escontext, yyscan_t yyscanner)
{
if (ch == 0)
{
@@ -586,14 +599,14 @@ addUnicodeChar(int ch, struct Node *escontext)
ereturn(escontext, false,
(errcode(ERRCODE_SYNTAX_ERROR),
errmsg("could not convert Unicode to
server encoding")));
- addstring(false, cbuf, strlen(cbuf));
+ addstring(false, cbuf, strlen(cbuf), yyscanner);
}
return true;
}
/* Add unicode character, processing any surrogate pairs */
static bool
-addUnicode(int ch, int *hi_surrogate, struct Node *escontext)
+addUnicode(int ch, int *hi_surrogate, struct Node *escontext, yyscan_t
yyscanner)
{
if (is_utf16_surrogate_first(ch))
{
@@ -626,7 +639,7 @@ addUnicode(int ch, int *hi_surrogate, struct Node
*escontext)
"surrogate.")));
}
- return addUnicodeChar(ch, escontext);
+ return addUnicodeChar(ch, escontext, yyscanner);
}
/*
@@ -664,7 +677,7 @@ parseUnicode(char *s, int l, struct Node *escontext,
yyscan_t yyscanner)
}
}
- if (! addUnicode(ch, &hi_surrogate, escontext))
+ if (! addUnicode(ch, &hi_surrogate, escontext, yyscanner))
return false;
}
@@ -692,7 +705,7 @@ parseHexChar(char *s, struct Node *escontext, yyscan_t
yyscanner)
ch = (s2 << 4) | s3;
- return addUnicodeChar(ch, escontext);
+ return addUnicodeChar(ch, escontext, yyscanner);
}
/*
--
2.47.1
