From 1a26845e138d3f26885f4d8cb5717f8db0b11988 Mon Sep 17 00:00:00 2001 From: Andrew Nicols Date: Wed, 10 Sep 2025 11:05:41 +0100 Subject: [PATCH 1/3] PHP 8.5 | Tokenizer/PHP: fix "Using null as an array offset" deprecation If a comment is on its own line, the new line token is merged into the comment token, and the new line is skipped by setting it to `null`. Where the next line contains incomplete, or invalid, code which ends in an nullsafe operator (for example `$obj?`), the tokenizer will step backwards until it finds the next non-empty line. The skipped new line token should be skipped during this parsing. This can only occur during live coding or when a file has a parse error, but PHPCS should handle that situation gracefully. Fixed now. Fixes #1216. This change is already covered via the existing tests. Ref: https://wiki.php.net/rfc/deprecations_php_8_5#deprecate_using_values_null_as_an_array_offset_and_when_calling_array_key_exists --- src/Tokenizers/PHP.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php index b5a7f84ada..bfb6cbb261 100644 --- a/src/Tokenizers/PHP.php +++ b/src/Tokenizers/PHP.php @@ -2255,6 +2255,9 @@ protected function tokenize($string) for ($i = ($stackPtr - 1); $i >= 0; $i--) { if (is_array($tokens[$i]) === true) { $tokenType = $tokens[$i][0]; + } else if ($tokens[$i] === null) { + // Ignore skipped tokens. + continue; } else { $tokenType = $tokens[$i]; } From c314822ea725cd87766b825570424fc8b8c3126b Mon Sep 17 00:00:00 2001 From: Andrew Nicols Date: Tue, 9 Sep 2025 14:03:28 +0100 Subject: [PATCH 2/3] Revert "PHP 8.5 | Tokenizer/PHP: temporarily silence "Using null as an array offset" deprecation" This reverts commit 6b82a86419bb51cdb25dc20d9ff56ea30f43a258 / PR 1215 as it is no longer needed. --- src/Tokenizers/PHP.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php index bfb6cbb261..f85c32125d 100644 --- a/src/Tokenizers/PHP.php +++ b/src/Tokenizers/PHP.php @@ -2271,7 +2271,7 @@ protected function tokenize($string) } if ($prevNonEmpty === null - && @isset(Tokens::$emptyTokens[$tokenType]) === false + && isset(Tokens::$emptyTokens[$tokenType]) === false ) { // Found the previous non-empty token. if ($tokenType === ':' || $tokenType === ',' || $tokenType === T_ATTRIBUTE_END) { @@ -2290,8 +2290,8 @@ protected function tokenize($string) if ($tokenType === T_FUNCTION || $tokenType === T_FN - || @isset(Tokens::$methodPrefixes[$tokenType]) === true - || @isset(Tokens::$scopeModifiers[$tokenType]) === true + || isset(Tokens::$methodPrefixes[$tokenType]) === true + || isset(Tokens::$scopeModifiers[$tokenType]) === true || $tokenType === T_VAR || $tokenType === T_READONLY ) { @@ -2314,7 +2314,7 @@ protected function tokenize($string) break; } - if (@isset(Tokens::$emptyTokens[$tokenType]) === false) { + if (isset(Tokens::$emptyTokens[$tokenType]) === false) { $lastSeenNonEmpty = $tokenType; } }//end for From b65945bb2905ffe887802b5c28962baf267df9e9 Mon Sep 17 00:00:00 2001 From: jrfnl Date: Mon, 3 Nov 2025 18:28:22 +0100 Subject: [PATCH 3/3] PHP 8.5 | Tokenizer/PHP: properly fix "Using null as an array offset" deprecation If a slash/hash comment is on its own line, the new line token is merged into the comment token for PHP 8.0+, and the new line is skipped by setting it to `null`. The PHP Tokenizer did not take the possibility of a token being set to `null` into account when retokening a `?` to either `T_NULLABLE` or `T_INLINE_THEN`, leading to the PHP 8.5 deprecation notice. In that case, the sniff first looks forward to see if we can draw a conclusion based on the non-empty tokens following the `?` and if not, walks backward to look at the tokens before the `?`. The problem occurs when a `null` token exists in the sequence before the `?`. This `null` token will only have been injected with a specific code sequence: when there is a slash/hash comment followed by a new line in the token sequence before the `?` and there is no indentation/new line whitespace on the next line. Also see the detailed analysis by andrewnicols in PHPCSStandards/PHP_CodeSniffer 1216#issuecomment-3274198443 There are only two situations in which this causes the tokenizer to hit the PHP 8.5 "Using null as an array offset" deprecation notice: 1. If the `?` token is the last token in the file (live coding - the issue was discovered via a test related to live coding). 2. If there are tokens after the `?`, but not tokens which allows us to draw a conclusion (yet) and there is a slash/hash comment between the `?` and the previous non-empty token. This commit: 1. Adds dedicated tests for both situations. 2. Adds a new fix for situation [1] as if there are no tokens after a `?`, we cannot determine definitively whether the `?` is a nullable operator or an inline then. For BC, this should be tokenized as `T_INLINE_THEN`. 3. Makes a small tweak to the previously added fix which addressed situation [2]. Alternatively, we could consider switching to using the "$finalTokens" token stack to do the token walking instead, but as that is a bigger change and this part of the code has nowhere near sufficient tests, that change is left for later. Fixes 1216 --- src/Tokenizers/PHP.php | 26 ++++++++++-- .../NullableVsInlineThenParseErrorTest.inc | 9 +++++ .../NullableVsInlineThenParseErrorTest.php | 40 +++++++++++++++++++ .../PHP/NullableVsInlineThenTest.inc | 6 ++- .../PHP/NullableVsInlineThenTest.php | 21 +++++----- 5 files changed, 88 insertions(+), 14 deletions(-) create mode 100644 tests/Core/Tokenizers/PHP/NullableVsInlineThenParseErrorTest.inc create mode 100644 tests/Core/Tokenizers/PHP/NullableVsInlineThenParseErrorTest.php diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php index f85c32125d..b45e37bceb 100644 --- a/src/Tokenizers/PHP.php +++ b/src/Tokenizers/PHP.php @@ -2244,6 +2244,24 @@ protected function tokenize($string) break; }//end for + // Handle live coding/parse errors elegantly. + // If the "?" is the last non-empty token in the file, we cannot draw a definitive conclusion, + // so tokenize as T_INLINE_THEN. + if ($i === $numTokens) { + if (PHP_CODESNIFFER_VERBOSITY > 1) { + echo "\t\t* token $stackPtr at end of file changed from ? to T_INLINE_THEN".PHP_EOL; + } + + $newToken['code'] = T_INLINE_THEN; + $newToken['type'] = 'T_INLINE_THEN'; + + $insideInlineIf[] = $stackPtr; + + $finalTokens[$newStackPtr] = $newToken; + $newStackPtr++; + continue; + } + /* * This can still be a nullable type or a ternary. * Do additional checking. @@ -2253,11 +2271,13 @@ protected function tokenize($string) $lastSeenNonEmpty = null; for ($i = ($stackPtr - 1); $i >= 0; $i--) { + if (isset($tokens[$i]) === false) { + // Ignore skipped tokens (related to PHP 8+ slash/hash comment vs new line retokenization). + continue; + } + if (is_array($tokens[$i]) === true) { $tokenType = $tokens[$i][0]; - } else if ($tokens[$i] === null) { - // Ignore skipped tokens. - continue; } else { $tokenType = $tokens[$i]; } diff --git a/tests/Core/Tokenizers/PHP/NullableVsInlineThenParseErrorTest.inc b/tests/Core/Tokenizers/PHP/NullableVsInlineThenParseErrorTest.inc new file mode 100644 index 0000000000..fda129b16b --- /dev/null +++ b/tests/Core/Tokenizers/PHP/NullableVsInlineThenParseErrorTest.inc @@ -0,0 +1,9 @@ +phpcsFile->getTokens(); + $target = $this->getTargetToken('/* testLiveCoding */', [T_NULLABLE, T_INLINE_THEN]); + $tokenArray = $tokens[$target]; + + $this->assertSame(T_INLINE_THEN, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_INLINE_THEN (code)'); + $this->assertSame('T_INLINE_THEN', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_INLINE_THEN (type)'); + + }//end testInlineThenAtEndOfFile() + + +}//end class diff --git a/tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.inc b/tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.inc index 398d2f6283..ac3c9f8b19 100644 --- a/tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.inc +++ b/tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.inc @@ -22,7 +22,11 @@ $closure = function ( /* testClosureParamTypeNullableInt */ ?Int $a, /* testClosureParamTypeNullableCallable */ - ? Callable $b + ? Callable $b, + /* testClosureParamTypeNullableStringWithAttributeAndSlashComment */ + #[AttributeForParam] + // This must be a slash or hash comment and the next line must **NOT** have any indentation for the PHP 8.5 deprecation notice (issue PHPCSStandards/PHP_CodeSniffer#1216) to occur. +?string $c /* testClosureReturnTypeNullableInt */ ) :?INT{}; diff --git a/tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.php b/tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.php index 81d4b6c01c..0b7e5d5510 100644 --- a/tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.php +++ b/tests/Core/Tokenizers/PHP/NullableVsInlineThenTest.php @@ -50,16 +50,17 @@ public function testNullable($testMarker) public static function dataNullable() { return [ - 'property declaration, readonly, no visibility' => ['/* testNullableReadonlyOnly */'], - 'property declaration, private set' => ['/* testNullablePrivateSet */'], - 'property declaration, public and protected set' => ['/* testNullablePublicProtectedSet */'], - 'property declaration, final, no visibility' => ['/* testNullableFinalOnly */'], - 'property declaration, abstract, no visibility' => ['/* testNullableAbstractOnly */'], - - 'closure param type, nullable int' => ['/* testClosureParamTypeNullableInt */'], - 'closure param type, nullable callable' => ['/* testClosureParamTypeNullableCallable */'], - 'closure return type, nullable int' => ['/* testClosureReturnTypeNullableInt */'], - 'function return type, nullable callable' => ['/* testFunctionReturnTypeNullableCallable */'], + 'property declaration, readonly, no visibility' => ['/* testNullableReadonlyOnly */'], + 'property declaration, private set' => ['/* testNullablePrivateSet */'], + 'property declaration, public and protected set' => ['/* testNullablePublicProtectedSet */'], + 'property declaration, final, no visibility' => ['/* testNullableFinalOnly */'], + 'property declaration, abstract, no visibility' => ['/* testNullableAbstractOnly */'], + + 'closure param type, nullable int' => ['/* testClosureParamTypeNullableInt */'], + 'closure param type, nullable callable' => ['/* testClosureParamTypeNullableCallable */'], + 'closure param type, nullable string with comment, issue #1216' => ['/* testClosureParamTypeNullableStringWithAttributeAndSlashComment */'], + 'closure return type, nullable int' => ['/* testClosureReturnTypeNullableInt */'], + 'function return type, nullable callable' => ['/* testFunctionReturnTypeNullableCallable */'], ]; }//end dataNullable()