Skip to content

Commit e862ab2

Browse files
freddenfredden
committed
Define polyfilled constants from Tokenizer as int
The value of these constants are not stable, and therefore already cannot be relied upon. This is because the specific values that PHP assigns can change with different versions of PHP. PHPCS does not use the values of these constants (other than to look up their name using the Tokens::tokenName() method). There are other tools which also polyfill these constants. Some of those tools also perform validation on the value for these constants. In order to play nicely with the arbitrary validation that other tools perform on these constants, we are switching from string values to integer values. The [PHP manual] suggests "using big numbers like 10000" for polyfilled T_* constants. We have arbitrarily chosen to start our numbering scheme from 135_000. All PHPCS 'native' tokens currently have reliable values. In line with PHP T_* constants, the values of these tokens should never be relied upon. In a future version of PHPCS, the values for these tokens will switch from strings to integers. Existing tests already cover the use of these constants and do not require adjustment for the code being changed here. [PHP manual]: https://www.php.net/manual/en/tokens.php
1 parent 892320f commit e862ab2

File tree

1 file changed

+63
-77
lines changed

1 file changed

+63
-77
lines changed

‎src/Util/Tokens.php‎

Lines changed: 63 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
namespace PHP_CodeSniffer\Util;
1212

13+
// PHPCS native tokens.
1314
define('T_NONE', 'PHPCS_T_NONE');
1415
define('T_OPEN_CURLY_BRACKET', 'PHPCS_T_OPEN_CURLY_BRACKET');
1516
define('T_CLOSE_CURLY_BRACKET', 'PHPCS_T_CLOSE_CURLY_BRACKET');
@@ -70,79 +71,6 @@
7071
define('T_TYPE_OPEN_PARENTHESIS', 'PHPCS_T_TYPE_OPEN_PARENTHESIS');
7172
define('T_TYPE_CLOSE_PARENTHESIS', 'PHPCS_T_TYPE_CLOSE_PARENTHESIS');
7273

73-
/*
74-
* {@internal IMPORTANT: all PHP native polyfilled tokens MUST be added to the
75-
* `PHP_CodeSniffer\Tests\Core\Util\Tokens\TokenNameTest::dataPolyfilledPHPNativeTokens()` test method!}
76-
*/
77-
78-
// Some PHP 7.4 tokens, replicated for lower versions.
79-
if (defined('T_COALESCE_EQUAL') === false) {
80-
define('T_COALESCE_EQUAL', 'PHPCS_T_COALESCE_EQUAL');
81-
}
82-
83-
if (defined('T_BAD_CHARACTER') === false) {
84-
define('T_BAD_CHARACTER', 'PHPCS_T_BAD_CHARACTER');
85-
}
86-
87-
if (defined('T_FN') === false) {
88-
define('T_FN', 'PHPCS_T_FN');
89-
}
90-
91-
// Some PHP 8.0 tokens, replicated for lower versions.
92-
if (defined('T_NULLSAFE_OBJECT_OPERATOR') === false) {
93-
define('T_NULLSAFE_OBJECT_OPERATOR', 'PHPCS_T_NULLSAFE_OBJECT_OPERATOR');
94-
}
95-
96-
if (defined('T_NAME_QUALIFIED') === false) {
97-
define('T_NAME_QUALIFIED', 'PHPCS_T_NAME_QUALIFIED');
98-
}
99-
100-
if (defined('T_NAME_FULLY_QUALIFIED') === false) {
101-
define('T_NAME_FULLY_QUALIFIED', 'PHPCS_T_NAME_FULLY_QUALIFIED');
102-
}
103-
104-
if (defined('T_NAME_RELATIVE') === false) {
105-
define('T_NAME_RELATIVE', 'PHPCS_T_NAME_RELATIVE');
106-
}
107-
108-
if (defined('T_MATCH') === false) {
109-
define('T_MATCH', 'PHPCS_T_MATCH');
110-
}
111-
112-
if (defined('T_ATTRIBUTE') === false) {
113-
define('T_ATTRIBUTE', 'PHPCS_T_ATTRIBUTE');
114-
}
115-
116-
// Some PHP 8.1 tokens, replicated for lower versions.
117-
if (defined('T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG') === false) {
118-
define('T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG', 'PHPCS_T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG');
119-
}
120-
121-
if (defined('T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG') === false) {
122-
define('T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG', 'PHPCS_T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG');
123-
}
124-
125-
if (defined('T_READONLY') === false) {
126-
define('T_READONLY', 'PHPCS_T_READONLY');
127-
}
128-
129-
if (defined('T_ENUM') === false) {
130-
define('T_ENUM', 'PHPCS_T_ENUM');
131-
}
132-
133-
// Some PHP 8.4 tokens, replicated for lower versions.
134-
if (defined('T_PUBLIC_SET') === false) {
135-
define('T_PUBLIC_SET', 'PHPCS_T_PUBLIC_SET');
136-
}
137-
138-
if (defined('T_PROTECTED_SET') === false) {
139-
define('T_PROTECTED_SET', 'PHPCS_T_PROTECTED_SET');
140-
}
141-
142-
if (defined('T_PRIVATE_SET') === false) {
143-
define('T_PRIVATE_SET', 'PHPCS_T_PRIVATE_SET');
144-
}
145-
14674
// Tokens used for parsing doc blocks.
14775
define('T_DOC_COMMENT_STAR', 'PHPCS_T_DOC_COMMENT_STAR');
14876
define('T_DOC_COMMENT_WHITESPACE', 'PHPCS_T_DOC_COMMENT_WHITESPACE');
@@ -158,6 +86,55 @@
15886
define('T_PHPCS_IGNORE', 'PHPCS_T_PHPCS_IGNORE');
15987
define('T_PHPCS_IGNORE_FILE', 'PHPCS_T_PHPCS_IGNORE_FILE');
16088

89+
/*
90+
* Polyfilled tokens.
91+
*
92+
* Unfortunately we can't easily put this block of code into the Tokens class
93+
* because we use these constants in public class constants there, so this code
94+
* needs to run before that class is ever initialised / referenced.
95+
*
96+
* {@internal IMPORTANT: all PHP native polyfilled tokens MUST be added to the
97+
* `PHP_CodeSniffer\Tests\Core\Util\Tokens\TokenNameTest::dataPolyfilledPHPNativeTokens()` test method!}
98+
*/
99+
100+
101+
$tokensToPolyfill = [
102+
'T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG',
103+
'T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG',
104+
'T_ATTRIBUTE',
105+
'T_BAD_CHARACTER',
106+
'T_COALESCE_EQUAL',
107+
'T_ENUM',
108+
'T_FN',
109+
'T_MATCH',
110+
'T_NAME_FULLY_QUALIFIED',
111+
'T_NAME_QUALIFIED',
112+
'T_NAME_RELATIVE',
113+
'T_NULLSAFE_OBJECT_OPERATOR',
114+
'T_PRIVATE_SET',
115+
'T_PROTECTED_SET',
116+
'T_PUBLIC_SET',
117+
'T_READONLY',
118+
];
119+
120+
$polyfillMappingTable = [];
121+
$nextTokenNumber = 135000;
122+
123+
foreach ($tokensToPolyfill as $tokenName) {
124+
if (defined($tokenName) === false) {
125+
while (isset($polyfillMappingTable[$nextTokenNumber]) === true) {
126+
$nextTokenNumber++;
127+
}
128+
129+
define($tokenName, $nextTokenNumber);
130+
}
131+
132+
$polyfillMappingTable[constant($tokenName)] = $tokenName;
133+
}
134+
135+
Tokens::$polyfillMappingTable = $polyfillMappingTable;
136+
unset($nextTokenNumber, $tokensToPolyfill, $polyfillMappingTable, $tokenName);
137+
161138
final class Tokens
162139
{
163140

@@ -606,6 +583,15 @@ final class Tokens
606583
T_YIELD_FROM => T_YIELD_FROM,
607584
];
608585

586+
/**
587+
* Mapping table for polyfilled constants
588+
*
589+
* {@internal This method is for internal use of PHPCS in the Tokens::tokenName() method.}
590+
*
591+
* @var array<int, string>
592+
*/
593+
public static $polyfillMappingTable = [];
594+
609595
/**
610596
* The token weightings.
611597
*
@@ -937,12 +923,12 @@ final class Tokens
937923
*/
938924
public static function tokenName($token)
939925
{
940-
if (is_string($token) === false) {
941-
// PHP-supplied token name.
942-
return token_name($token);
926+
if (is_string($token) === true) {
927+
// PHPCS native token.
928+
return substr($token, 6);
943929
}
944930

945-
return substr($token, 6);
931+
return (self::$polyfillMappingTable[$token] ?? token_name($token));
946932
}
947933

948934

0 commit comments

Comments
 (0)