Skip to content

Zend language scanner: minor refactorings #20480

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+29 −21
Open

Zend language scanner: minor refactorings #20480

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9254d52
Use uint32_t type
Girgias Nov 13, 2025
b613995
Make some structs private
Girgias Nov 14, 2025
16665ef
Add some header includes and detail reasons why
Girgias Nov 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 8 additions & 14 deletions Zend/zend_language_scanner.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,14 @@
#ifndef ZEND_SCANNER_H
#define ZEND_SCANNER_H

#include "zend_globals.h"
#include "zend_arena.h"
Copy link
Member

@ndossche ndossche Nov 15, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You still include zend_globals, so what exactly do we gain by adding all these headers explicitly?

Copy link
Member Author

@Girgias Girgias Nov 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wanted to remove it, but annoyingly zend_php_scanner_event is defined there. And I don't know if it's sensible to "flip" the header inclusion so that zend_globals includes zend_language_scanner.h or not...

Copy link
Member

@ndossche ndossche Nov 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm I don't think that makes sense, I don't like to pull the scanner into everything

Copy link
Member Author

@Girgias Girgias Nov 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's what I thought too, I'm happy to remove the extra header includes and just have a comment explaining why we include globals.

Copy link
Member

@ndossche ndossche Nov 17, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's better

#include "zend_ast.h"
#include "zend_stack.h"
#include "zend_ptr_stack.h"
#include "zend_stream.h" /* For zend_file_handle */
#include "zend_string.h"
#include "zend_multibyte.h" /* For zend_encoding_filter and zend_encoding */
#include "zend_globals.h" /* For zend_php_scanner_event */

typedef struct _zend_lex_state {
unsigned int yy_leng;
Expand Down Expand Up @@ -61,19 +68,6 @@ typedef struct _zend_lex_state {
zend_arena *ast_arena;
} zend_lex_state;

typedef struct _zend_heredoc_label {
Copy link
Member

@ndossche ndossche Nov 15, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think it makes sense to make these private, as they can be accessed via the public struct via heredoc_label_stack. Similar issue for zend_nest_location.

Copy link
Member Author

@Girgias Girgias Nov 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point, will revert.

char *label;
int length;
int indentation;
bool indentation_uses_spaces;
} zend_heredoc_label;

/* Track locations of unclosed {, [, (, etc. for better syntax error reporting */
typedef struct _zend_nest_location {
char text;
int lineno;
} zend_nest_location;

BEGIN_EXTERN_C()
ZEND_API void zend_save_lexical_state(zend_lex_state *lex_state);
ZEND_API void zend_restore_lexical_state(zend_lex_state *lex_state);
Expand Down
28 changes: 21 additions & 7 deletions Zend/zend_language_scanner.l
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include "zend_language_scanner_defs.h"

#include <errno.h>
#include <stdint.h>
#include "zend.h"
#ifdef ZEND_WIN32
# include <Winuser.h>
Expand All @@ -47,6 +48,19 @@
#include "zend_exceptions.h"
#include "zend_virtual_cwd.h"

typedef struct _zend_heredoc_label {
char *label;
int length;
int indentation;
bool indentation_uses_spaces;
} zend_heredoc_label;

/* Track locations of unclosed {, [, (, etc. for better syntax error reporting */
typedef struct _zend_nest_location {
char text;
uint32_t lineno;
} zend_nest_location;

#define YYCTYPE unsigned char
#define YYFILL(n) { if ((YYCURSOR + n) >= (YYLIMIT + ZEND_MMAP_AHEAD)) { return 0; } }
#define YYCURSOR SCNG(yy_cursor)
Expand Down Expand Up @@ -600,7 +614,7 @@ static zend_op_array *zend_compile(int type)
CG(ast_arena) = zend_arena_create(1024 * 32);

if (!zendparse()) {
int last_lineno = CG(zend_lineno);
uint32_t last_lineno = CG(zend_lineno);
zend_file_context original_file_context;
zend_oparray_context original_oparray_context;
zend_op_array *original_active_op_array = CG(active_op_array);
Expand Down Expand Up @@ -1140,7 +1154,7 @@ skip_escape_conversion:
unsigned char *str;
// TODO: avoid realocation ???
s = Z_STRVAL_P(zendlval);
SCNG(output_filter)(&str, &sz, (unsigned char *)s, (size_t)Z_STRLEN_P(zendlval));
SCNG(output_filter)(&str, &sz, (unsigned char *)s, Z_STRLEN_P(zendlval));
zval_ptr_dtor(zendlval);
ZVAL_STRINGL(zendlval, (char *) str, sz);
efree(str);
Expand Down Expand Up @@ -1172,7 +1186,7 @@ static bool strip_multiline_string_indentation(
const char *str = Z_STRVAL_P(zendlval), *end = str + Z_STRLEN_P(zendlval);
char *copy = Z_STRVAL_P(zendlval);

int newline_count = 0;
uint32_t newline_count = 0;
size_t newline_len;
const char *nl;

Expand Down Expand Up @@ -1253,7 +1267,7 @@ static void copy_heredoc_label_stack(void *void_heredoc_label)
}

/* Check that { }, [ ], ( ) are nested correctly */
static void report_bad_nesting(char opening, int opening_lineno, char closing)
static void report_bad_nesting(char opening, uint32_t opening_lineno, char closing)
{
char buf[256];
size_t used = 0;
Expand Down Expand Up @@ -1361,7 +1375,7 @@ int ZEND_FASTCALL lex_scan(zval *zendlval, zend_parser_stack_elem *elem)
{
int token;
int offset;
int start_line = CG(zend_lineno);
uint32_t start_line = CG(zend_lineno);

ZVAL_UNDEF(zendlval);
restart:
Expand Down Expand Up @@ -2499,7 +2513,7 @@ inline_char_handler:
if (YYCURSOR < YYLIMIT) {
YYCURSOR++;
} else {
zend_throw_exception_ex(zend_ce_parse_error, 0, "Unterminated comment starting line %d", CG(zend_lineno));
zend_throw_exception_ex(zend_ce_parse_error, 0, "Unterminated comment starting line %" PRIu32, CG(zend_lineno));
if (PARSER_MODE()) {
RETURN_TOKEN(T_ERROR);
}
Expand Down Expand Up @@ -2616,7 +2630,7 @@ skip_escape_conversion:
zend_string *new_str;
s = Z_STRVAL_P(zendlval);
// TODO: avoid reallocation ???
SCNG(output_filter)((unsigned char **)&str, &sz, (unsigned char *)s, (size_t)Z_STRLEN_P(zendlval));
SCNG(output_filter)((unsigned char **)&str, &sz, (unsigned char *)s, Z_STRLEN_P(zendlval));
new_str = zend_string_init(str, sz, 0);
if (str != s) {
efree(str);
Expand Down