From 15634531e9dd651e4bb9b3fff889ad03be977e92 Mon Sep 17 00:00:00 2001 From: Jackson Cooper Date: Fri, 13 Feb 2026 23:23:16 +1100 Subject: [PATCH] feat: glob pattern support in executionEnvironments root and extraPaths --- docs/configuration/config-files.md | 16 ++-- .../src/analyzer/backgroundAnalysisProgram.ts | 3 +- .../src/analyzer/importResolver.ts | 2 +- .../src/backgroundAnalysisBase.ts | 16 ++-- .../src/common/configOptions.ts | 44 +++++++++-- .../pyright-internal/src/tests/config.test.ts | 78 ++++++++++++++++++- .../src/tests/importResolver.test.ts | 75 +++++++++++++++++- .../schemas/pyrightconfig.schema.json | 7 +- 8 files changed, 209 insertions(+), 32 deletions(-) diff --git a/docs/configuration/config-files.md b/docs/configuration/config-files.md index bcb0a11763..a5040c7034 100644 --- a/docs/configuration/config-files.md +++ b/docs/configuration/config-files.md @@ -30,7 +30,7 @@ The following settings control the *environment* in which basedpyright will chec - **verboseOutput** [boolean]: Specifies whether output logs should be verbose. This is useful when diagnosing certain problems like import resolution issues. -- **extraPaths** [array of strings, optional]: Additional search paths that will be used when searching for modules imported by files. +- **extraPaths** [array of strings, optional]: Additional search paths that will be used when searching for modules imported by files. Glob patterns (`**`, `*`, `?`) are supported *(basedpyright exclusive)* — matching directories are expanded at configuration time. - **pythonVersion** [string, optional]: Specifies the version of Python that will be used to execute the source code. The version should be specified as a string in the format "M.m" where M is the major version and m is the minor (e.g. `"3.0"` or `"3.6"`). If a version is provided, pyright will generate errors if the source code makes use of language features that are not supported in that version. It will also tailor its use of type stub files, which conditionalizes type definitions based on the version. If no version is specified, pyright will use the version of the current python interpreter, if one is present. @@ -314,13 +314,13 @@ The following settings allow more fine grained control over the **typeCheckingMo ## Execution Environment Options -Pyright allows multiple “execution environments” to be defined for different portions of your source tree. For example, a subtree may be designed to run with different import search paths or a different version of the python interpreter than the rest of the source base. +Pyright allows multiple "execution environments" to be defined for different portions of your source tree. For example, a subtree may be designed to run with different import search paths or a different version of the python interpreter than the rest of the source base. -The following settings can be specified for each execution environment. Each source file within a project is associated with at most one execution environment -- the first one whose root directory contains that file. +The following settings can be specified for each execution environment. Each source file within a project is associated with at most one execution environment -- the first one whose root matches that file. Environments are searched in array order; the first match wins. -- **root** [string, required]: Root path for the code that will execute within this execution environment. +- **root** [string, required]: Root path for the code that will execute within this execution environment. Glob patterns (`**`, `*`, `?`) are supported *(basedpyright exclusive)* — when used, import resolution falls back to the project root. -- **extraPaths** [array of strings, optional]: Additional search paths (in addition to the root path) that will be used when searching for modules imported by files within this execution environment. If specified, this overrides the default extraPaths setting when resolving imports for files within this execution environment. Note that each file’s execution environment mapping is independent, so if file A is in one execution environment and imports a second file B within a second execution environment, any imports from B will use the extraPaths in the second execution environment. +- **extraPaths** [array of strings, optional]: Additional search paths (in addition to the root path) that will be used when searching for modules imported by files within this execution environment. Glob patterns (`**`, `*`, `?`) are supported *(basedpyright exclusive)* — matching directories are expanded at configuration time. If specified, this overrides the default extraPaths setting when resolving imports for files within this execution environment. Note that each file's execution environment mapping is independent, so if file A is in one execution environment and imports a second file B within a second execution environment, any imports from B will use the extraPaths in the second execution environment. - **pythonVersion** [string, optional]: The version of Python used for this execution environment. If not specified, the global `pythonVersion` setting is used instead. @@ -377,10 +377,10 @@ The following is an example of a pyright config file: ] }, { - "root": "src/tests", + "root": "**/tests", "reportPrivateUsage": false, "extraPaths": [ - "src/tests/e2e", + "**/fixtures", "src/sdk" ] }, @@ -413,7 +413,7 @@ pythonPlatform = "Linux" executionEnvironments = [ { root = "src/web", pythonVersion = "3.5", pythonPlatform = "Windows", extraPaths = [ "src/service_libs" ], reportMissingImports = "warning" }, { root = "src/sdk", pythonVersion = "3.0", extraPaths = [ "src/backend" ] }, - { root = "src/tests", reportPrivateUsage = false, extraPaths = ["src/tests/e2e", "src/sdk" ]}, + { root = "**/tests", reportPrivateUsage = false, extraPaths = ["**/fixtures", "src/sdk"] }, { root = "src" } ] ``` diff --git a/packages/pyright-internal/src/analyzer/backgroundAnalysisProgram.ts b/packages/pyright-internal/src/analyzer/backgroundAnalysisProgram.ts index 424a8ba8ec..214e22f437 100644 --- a/packages/pyright-internal/src/analyzer/backgroundAnalysisProgram.ts +++ b/packages/pyright-internal/src/analyzer/backgroundAnalysisProgram.ts @@ -277,7 +277,8 @@ export class BackgroundAnalysisProgram { } private _ensurePartialStubPackages(execEnv: ExecutionEnvironment) { - this._backgroundAnalysis?.ensurePartialStubPackages(execEnv.root?.toString()); + const execEnvIndex = this.configOptions.getExecutionEnvironments().indexOf(execEnv); + this._backgroundAnalysis?.ensurePartialStubPackages(execEnvIndex); return this._importResolver.ensurePartialStubPackages(execEnv); } diff --git a/packages/pyright-internal/src/analyzer/importResolver.ts b/packages/pyright-internal/src/analyzer/importResolver.ts index 0fb48958a8..000cb86a2c 100644 --- a/packages/pyright-internal/src/analyzer/importResolver.ts +++ b/packages/pyright-internal/src/analyzer/importResolver.ts @@ -413,7 +413,7 @@ export class ImportResolver { return false; } - if (this.partialStubs.isPartialStubPackagesScanned(execEnv)) { + if (execEnv.root && this.partialStubs.isPathScanned(execEnv.root)) { return false; } diff --git a/packages/pyright-internal/src/backgroundAnalysisBase.ts b/packages/pyright-internal/src/backgroundAnalysisBase.ts index 87b91e3663..50f9735d47 100644 --- a/packages/pyright-internal/src/backgroundAnalysisBase.ts +++ b/packages/pyright-internal/src/backgroundAnalysisBase.ts @@ -55,7 +55,7 @@ export interface IBackgroundAnalysis extends Disposable { setConfigOptions(configOptions: ConfigOptions): void; setTrackedFiles(fileUris: Uri[]): void; setAllowedThirdPartyImports(importNames: string[]): void; - ensurePartialStubPackages(executionRoot: string | undefined): void; + ensurePartialStubPackages(execEnvIndex: number): void; setFileOpened(fileUri: Uri, version: number | null, contents: string, options: OpenFileOptions): void; updateChainedUri(fileUri: Uri, chainedUri: Uri | undefined): void; setFileClosed(fileUri: Uri, isTracked?: boolean): void; @@ -158,8 +158,8 @@ export class BackgroundAnalysisBase implements IBackgroundAnalysis { this.enqueueRequest({ requestType: 'setAllowedThirdPartyImports', data: serialize(importNames) }); } - ensurePartialStubPackages(executionRoot: string | undefined) { - this.enqueueRequest({ requestType: 'ensurePartialStubPackages', data: serialize({ executionRoot }) }); + ensurePartialStubPackages(execEnvIndex: number) { + this.enqueueRequest({ requestType: 'ensurePartialStubPackages', data: serialize({ execEnvIndex }) }); } setFileOpened(fileUri: Uri, version: number | null, contents: string, options: OpenFileOptions) { @@ -589,8 +589,8 @@ export abstract class BackgroundAnalysisRunnerBase extends BackgroundThreadBase } case 'ensurePartialStubPackages': { - const { executionRoot } = deserialize(msg.data); - this.handleEnsurePartialStubPackages(executionRoot); + const { execEnvIndex } = deserialize(msg.data); + this.handleEnsurePartialStubPackages(execEnvIndex); break; } @@ -775,10 +775,8 @@ export abstract class BackgroundAnalysisRunnerBase extends BackgroundThreadBase this.program.setAllowedThirdPartyImports(importNames); } - protected handleEnsurePartialStubPackages(executionRoot: string | undefined) { - const execEnv = this._configOptions - .getExecutionEnvironments() - .find((e) => e.root?.toString() === executionRoot); + protected handleEnsurePartialStubPackages(execEnvIndex: number) { + const execEnv = this._configOptions.getExecutionEnvironments()[execEnvIndex]; if (execEnv) { this.importResolver.ensurePartialStubPackages(execEnv); } diff --git a/packages/pyright-internal/src/common/configOptions.ts b/packages/pyright-internal/src/common/configOptions.ts index 6dbf559fa5..94e6a85488 100644 --- a/packages/pyright-internal/src/common/configOptions.ts +++ b/packages/pyright-internal/src/common/configOptions.ts @@ -7,6 +7,9 @@ * Class that holds the configuration options for the analyzer. */ +import { globSync } from 'node:fs'; +import { matchesGlob } from 'node:path/posix'; + import { ImportLogger } from '../analyzer/importLogger'; import { getPathsFromPthFiles } from '../analyzer/pythonPathUtils'; import * as pathConsts from '../common/pathConsts'; @@ -27,7 +30,7 @@ import { PythonVersion, latestStablePythonVersion } from './pythonVersion'; import { ServiceKeys } from './serviceKeys'; import { ServiceProvider } from './serviceProvider'; import { Uri } from './uri/uri'; -import { FileSpec, getFileSpec, isDirectory } from './uri/uriUtils'; +import { FileSpec, getFileSpec, getWildcardRoot, isDirectory } from './uri/uriUtils'; import { userFacingOptionsList } from './stringUtils'; // prevent upstream changes from sneaking in and adding errors using console.error, @@ -66,6 +69,8 @@ export class ExecutionEnvironment { // tools or playgrounds). skipNativeLibraries: boolean; + rootGlob?: string; + // Default to "." which indicates every file in the project. constructor( name: string, @@ -1526,13 +1531,21 @@ export class ConfigOptions { // execution environment is used. findExecEnvironment(file: Uri): ExecutionEnvironment { return ( - this.executionEnvironments.find((env) => { - const envRoot = Uri.is(env.root) ? env.root : this.projectRoot.resolvePaths(env.root || ''); - return file.startsWith(envRoot); - }) ?? this.getDefaultExecEnvironment() + this.executionEnvironments.find((env) => this._fileMatchesEnvironment(file, env)) ?? + this.getDefaultExecEnvironment() ); } + private _fileMatchesEnvironment(file: Uri, env: ExecutionEnvironment): boolean { + if (env.rootGlob !== undefined) { + const relative = this.projectRoot.getRelativePath(file)?.slice(2); + if (relative === undefined) return false; + return matchesGlob(relative, env.rootGlob + '/**'); + } + const envRoot = Uri.is(env.root) ? env.root : this.projectRoot.resolvePaths(env.root || ''); + return file.startsWith(envRoot); + } + getExecutionEnvironments(): ExecutionEnvironment[] { if (this.executionEnvironments.length > 0) { return this.executionEnvironments; @@ -1691,7 +1704,7 @@ export class ConfigOptions { if (typeof path !== 'string') { console.error(`Config "extraPaths" field ${pathIndex} must be a string.`); } else { - configExtraPaths!.push(configDirUri.resolvePaths(path)); + this._resolveExtraPath(path, configDirUri, configExtraPaths, console); } }); this.defaultExtraPaths = [...configExtraPaths]; @@ -1989,6 +2002,20 @@ export class ConfigOptions { return this.pythonEnvironmentName || this.pythonPath?.toString() || 'python'; } + private _resolveExtraPath(path: string, configDirUri: Uri, out: Uri[], console: ConsoleInterface) { + if (/[*?]/.test(path)) { + try { + for (const match of globSync(path, { cwd: configDirUri.getFilePath() })) { + out.push(configDirUri.resolvePaths(match)); + } + } catch (e) { + console.error(`Failed to expand glob pattern "${path}": ${e}`); + } + } else { + out.push(configDirUri.resolvePaths(path)); + } + } + private _convertBoolean(value: any, fieldName: string, defaultValue: boolean): boolean { if (value === undefined) { return defaultValue; @@ -2043,7 +2070,8 @@ export class ConfigOptions { // Validate the root. if (envObj.root && typeof envObj.root === 'string') { - newExecEnv.root = configDirUri.resolvePaths(envObj.root); + newExecEnv.rootGlob = envObj.root; + newExecEnv.root = getWildcardRoot(configDirUri, envObj.root); } else { console.error(`Config executionEnvironments index ${index}: missing root value.`); } @@ -2067,7 +2095,7 @@ export class ConfigOptions { ` extraPaths field ${pathIndex} must be a string.` ); } else { - newExecEnv.extraPaths.push(configDirUri.resolvePaths(path)); + this._resolveExtraPath(path, configDirUri, newExecEnv.extraPaths, console); } }); } diff --git a/packages/pyright-internal/src/tests/config.test.ts b/packages/pyright-internal/src/tests/config.test.ts index 7589ca12e1..d37e67c4b4 100644 --- a/packages/pyright-internal/src/tests/config.test.ts +++ b/packages/pyright-internal/src/tests/config.test.ts @@ -12,7 +12,11 @@ import assert from 'assert'; import { AnalyzerService } from '../analyzer/service'; import { deserialize, serialize } from '../backgroundThreadBase'; import { CommandLineOptions, DiagnosticSeverityOverrides } from '../common/commandLineOptions'; -import { ConfigOptions, ExecutionEnvironment, getStandardDiagnosticRuleSet } from '../common/configOptions'; +import { + ConfigOptions, + ExecutionEnvironment, + getStandardDiagnosticRuleSet, +} from '../common/configOptions'; import { ConsoleInterface, NullConsole } from '../common/console'; import { TaskListPriority } from '../common/diagnostic'; import { combinePaths, normalizePath, normalizeSlashes } from '../common/pathUtils'; @@ -740,4 +744,76 @@ describe(`config test'}`, () => { shouldRunAnalysis: () => true, }); } + + describe('glob root support', () => { + function setupExecEnvConfig(roots: ({ root: string } & Record)[]) { + const cwd = UriEx.file(normalizePath(process.cwd())); + const configOptions = new ConfigOptions(cwd); + const json = { executionEnvironments: roots }; + const fs = new TestFileSystem(false); + const console = new ErrorTrackingNullConsole(); + const sp = createServiceProvider(fs, console); + configOptions.initializeFromJson(json, cwd, sp, new NoAccessHost()); + configOptions.setupExecutionEnvironments(json, cwd, console); + return { cwd, configOptions, console }; + } + + test.each([ + 'src', '**/tests', 'src/*/utils', 'src/test?', '***/tests', ' ', + ])('root "%s" sets rootGlob', (root) => { + const { configOptions, console } = setupExecEnvConfig([{ root }]); + assert.deepStrictEqual(console.errors, []); + const env = configOptions.executionEnvironments[0]; + assert.ok(env); + assert.strictEqual(env.rootGlob, root); + assert.ok(env.root); + }); + + test('serialization round-trip preserves rootGlob', () => { + const { configOptions } = setupExecEnvConfig([{ root: '**/tests' }]); + const cloned = deserialize(serialize(configOptions)); + assert.strictEqual( + cloned.executionEnvironments[0].rootGlob, + configOptions.executionEnvironments[0].rootGlob + ); + }); + + test.each([ + { roots: ['**/tests'], file: 'tests/test_foo.py', envIndex: 0 }, + { roots: ['**/tests'], file: 'src/tests/test_foo.py', envIndex: 0 }, + { roots: ['**/tests'], file: 'src/lib/deep/tests/test_bar.py', envIndex: 0 }, + { roots: ['**/tests'], file: 'src/testing/foo.py', envIndex: -1 }, + { roots: ['src/*/utils'], file: 'src/foo/utils/helper.py', envIndex: 0 }, + { roots: ['src/*/utils'], file: 'src/foo/bar/utils/helper.py', envIndex: -1 }, + { roots: ['src/v?/lib'], file: 'src/v1/lib/foo.py', envIndex: 0 }, + { roots: ['src/v?/lib'], file: 'src/v10/lib/foo.py', envIndex: -1 }, + { roots: ['src/core', '**/tests'], file: 'src/core/tests/test_foo.py', envIndex: 0 }, + { roots: ['src/core', '**/tests'], file: 'lib/tests/test_bar.py', envIndex: 1 }, + { roots: ['src/core', '**/tests'], file: 'other/module.py', envIndex: -1 }, + { roots: ['**/tests', 'src'], file: 'src/tests/test_foo.py', envIndex: 0 }, + { roots: ['**/tests', 'src'], file: 'src/module.py', envIndex: 1 }, + { roots: ['**/tests', '**/test'], file: 'src/tests/foo.py', envIndex: 0 }, + { roots: ['**/tests', '**/test'], file: 'src/test/foo.py', envIndex: 1 }, + { roots: ['**/tests', '**/test'], file: 'src/tests/test/foo.py', envIndex: 0 }, + ])('roots $roots: "$file" -> env $envIndex', ({ roots, file, envIndex }) => { + const { cwd, configOptions } = setupExecEnvConfig(roots.map((root) => ({ root }))); + const result = configOptions.findExecEnvironment(cwd.resolvePaths(file)); + if (envIndex >= 0) { + assert.strictEqual(result, configOptions.executionEnvironments[envIndex]); + } else { + for (const env of configOptions.executionEnvironments) { + assert.notStrictEqual(result, env); + } + } + }); + + test('glob root with diagnostic overrides', () => { + const { configOptions } = setupExecEnvConfig([ + { root: '**/tests', reportPrivateUsage: false }, + ]); + const env = configOptions.executionEnvironments[0]; + assert.strictEqual(env.root!.key, configOptions.projectRoot.key); + assert.strictEqual(env.diagnosticRuleSet.reportPrivateUsage, 'none'); + }); + }); }); diff --git a/packages/pyright-internal/src/tests/importResolver.test.ts b/packages/pyright-internal/src/tests/importResolver.test.ts index c0a2898445..3a846194b4 100644 --- a/packages/pyright-internal/src/tests/importResolver.test.ts +++ b/packages/pyright-internal/src/tests/importResolver.test.ts @@ -10,7 +10,7 @@ import { Dirent, ReadStream, WriteStream } from 'fs'; import { Disposable } from 'vscode-jsonrpc'; import { ImportResolver } from '../analyzer/importResolver'; import { ImportType } from '../analyzer/importResult'; -import { ConfigOptions } from '../common/configOptions'; +import { ConfigOptions, ExecutionEnvironment } from '../common/configOptions'; import { FileSystem, MkDirOptions, Stats } from '../common/fileSystem'; import { FileWatcher, FileWatcherEventHandler } from '../common/fileWatcher'; import { FullAccessHost } from '../common/fullAccessHost'; @@ -22,7 +22,7 @@ import { ServiceKeys } from '../common/serviceKeys'; import { ServiceProvider } from '../common/serviceProvider'; import { createServiceProvider } from '../common/serviceProviderExtensions'; import { Uri } from '../common/uri/uri'; -import { UriEx } from '../common/uri/uriUtils'; +import { UriEx, getWildcardRoot } from '../common/uri/uriUtils'; import { PartialStubService } from '../partialStubService'; import { PyrightFileSystem } from '../pyrightFileSystem'; import { TestAccessHost } from './harness/testAccessHost'; @@ -824,6 +824,77 @@ describe('Import tests with fake venv', () => { }); } + describe('glob-root execution environment import resolution', () => { + test('glob root resolves from project root', () => { + const files = [ + { path: normalizeSlashes('/mylib/__init__.py'), content: '' }, + { path: normalizeSlashes('/mylib/core.py'), content: 'x = 1' }, + { path: normalizeSlashes('/tests/test_core.py'), content: '' }, + ]; + + const result = getImportResult(files, ['mylib', 'core'], (c) => { + const env = new ExecutionEnvironment( + 'tests', + c.projectRoot, + c.diagnosticRuleSet, + undefined, + undefined, + undefined + ); + env.rootGlob = '**/tests'; + env.root = getWildcardRoot(c.projectRoot, '**/tests'); + c.executionEnvironments = [env]; + }); + + assert(result.isImportFound, `Import not found: ${result.importFailureInfo?.join('\n')}`); + }); + + test('plain root resolves from own root', () => { + const files = [ + { path: normalizeSlashes('/src/__init__.py'), content: '' }, + { path: normalizeSlashes('/src/module.py'), content: 'x = 1' }, + { path: normalizeSlashes('/src/tests/test_module.py'), content: '' }, + ]; + + const result = getImportResult(files, ['module'], (c) => { + const env = new ExecutionEnvironment( + 'src', + UriEx.file(normalizeSlashes('/src')), + c.diagnosticRuleSet, + undefined, + undefined, + undefined + ); + c.executionEnvironments = [env]; + }); + + assert(result.isImportFound, `Import not found: ${result.importFailureInfo?.join('\n')}`); + }); + + test('glob root module name from project root', () => { + const files = [ + { path: normalizeSlashes('/mylib/__init__.py'), content: '' }, + { path: normalizeSlashes('/mylib/core.py'), content: '' }, + ]; + + const result = getModuleNameForImport(files, (c) => { + const env = new ExecutionEnvironment( + 'mylib', + c.projectRoot, + c.diagnosticRuleSet, + undefined, + undefined, + undefined + ); + env.rootGlob = '**/mylib'; + env.root = getWildcardRoot(c.projectRoot, '**/mylib'); + c.executionEnvironments = [env]; + }); + + assert.strictEqual(result.moduleName, 'mylib.core'); + }); + }); + function getImportResult( files: { path: string; content: string }[], nameParts: string[], diff --git a/packages/vscode-pyright/schemas/pyrightconfig.schema.json b/packages/vscode-pyright/schemas/pyrightconfig.schema.json index 23348336d9..163ae3f5b6 100644 --- a/packages/vscode-pyright/schemas/pyrightconfig.schema.json +++ b/packages/vscode-pyright/schemas/pyrightconfig.schema.json @@ -25,6 +25,7 @@ "extraPaths": { "type": "array", "title": "Additional import search resolution paths", + "description": "Glob patterns (**, *, ?) are supported and expanded at configuration time.", "items": { "type": "string", "title": "Additional import search resolution path", @@ -1027,9 +1028,11 @@ "properties": { "root": { "type": "string", - "title": "Path to code subdirectory to which these settings apply", + "title": "Path or glob pattern for code to which these settings apply", + "description": "Root path or glob pattern (**, *, ?). Glob patterns use the project root for import resolution. (basedpyright exclusive)", "default": "", - "pattern": "^(.*)$" + "pattern": "^(.*)$", + "examples": ["src", "**/tests", "src/*/utils"] }, "disableBytesTypePromotions": { "$ref": "#/definitions/disableBytesTypePromotions"