From 2182cd54b95857310ca9f9253e1294fe6a44cfa7 Mon Sep 17 00:00:00 2001 From: ianhelle Date: Tue, 21 Oct 2025 13:11:46 -0700 Subject: [PATCH 1/8] Fix for removal of Defender XDR endpoint - also fixing datetime formatting change for AzureMonitor - removing spurious warning when using Default MicrosoftDefender configuration in msticpyconfig.yaml --- msticpy/_version.py | 2 +- msticpy/data/core/query_source.py | 2 +- msticpy/data/drivers/azure_monitor_driver.py | 8 +----- msticpy/data/drivers/mdatp_driver.py | 26 +++++++++----------- msticpy/data/drivers/odata_driver.py | 6 ++++- tests/data/drivers/test_mdatp_driver.py | 5 ++-- tests/data/drivers/test_odata_drivers.py | 14 +++++------ 7 files changed, 29 insertions(+), 34 deletions(-) diff --git a/msticpy/_version.py b/msticpy/_version.py index 3ec49d68..bf65a81f 100644 --- a/msticpy/_version.py +++ b/msticpy/_version.py @@ -1,3 +1,3 @@ """Version file.""" -VERSION = "2.17.1" +VERSION = "2.17.2" diff --git a/msticpy/data/core/query_source.py b/msticpy/data/core/query_source.py index a8da146d..f6264192 100644 --- a/msticpy/data/core/query_source.py +++ b/msticpy/data/core/query_source.py @@ -27,7 +27,7 @@ def _value_or_default(src_dict: dict, prop_name: str, default: dict): - """Return value from dict or emtpy dict.""" + """Return value from dict or empty dict.""" src_value = src_dict.get(prop_name) return src_value if src_value is not None else default diff --git a/msticpy/data/drivers/azure_monitor_driver.py b/msticpy/data/drivers/azure_monitor_driver.py index 9b13f8f0..984d3d73 100644 --- a/msticpy/data/drivers/azure_monitor_driver.py +++ b/msticpy/data/drivers/azure_monitor_driver.py @@ -19,7 +19,6 @@ import contextlib import logging import warnings -from datetime import datetime from typing import Any, Iterable, cast import httpx @@ -112,7 +111,7 @@ def __init__(self, connection_str: str | None = None, **kwargs): self._schema: dict[str, Any] = {} self.set_driver_property( DriverProps.FORMATTERS, - {"datetime": self._format_datetime, "list": self._format_list}, + {"list": self._format_list}, ) self._loaded = True self._ua_policy = UserAgentPolicy(user_agent=mp_ua_header()["UserAgent"]) @@ -639,11 +638,6 @@ def _get_schema(self) -> dict[str, dict]: ) return _schema_format_tables(tables) - @staticmethod - def _format_datetime(date_time: datetime) -> str: - """Return datetime-formatted string.""" - return date_time.isoformat(sep="T") + "Z" - @staticmethod def _format_list(param_list: Iterable[Any]): """Return formatted list parameter.""" diff --git a/msticpy/data/drivers/mdatp_driver.py b/msticpy/data/drivers/mdatp_driver.py index 0cd2ac4f..03e93113 100644 --- a/msticpy/data/drivers/mdatp_driver.py +++ b/msticpy/data/drivers/mdatp_driver.py @@ -16,11 +16,7 @@ from ..._version import VERSION from ...auth.azure_auth_core import AzureCloudConfig -from ...auth.cloud_mappings import ( - get_defender_endpoint, - get_m365d_endpoint, - get_m365d_login_endpoint, -) +from ...auth.cloud_mappings import get_defender_endpoint, get_m365d_login_endpoint from ...common.data_utils import ensure_df_datetimes from ...common.utility import export from ..core.query_defns import DataEnvironment @@ -117,7 +113,7 @@ def __init__( self.cloud: str = cs_dict.pop("cloud", "global") if cloud: - logger.info("Overriding configured cloud with: %s", cloud) + logger.info("Using Azure cloud: %s", cloud) self.cloud = cloud else: logger.debug("Using cloud from configuration: %s", self.cloud) @@ -290,16 +286,16 @@ def _select_api(data_environment: DataEnvironment, cloud: str) -> M365DConfigura resource_uri: str = az_cloud_config.endpoints["microsoftGraphResourceId"] api_version = "v1.0" api_endpoint = "/security/runHuntingQuery" - elif data_environment == DataEnvironment.M365D: - logger.info("Using M365 Defender Advanced Hunting API") - login_uri = urljoin( - get_m365d_login_endpoint(cloud), - "{tenantId}/oauth2/v2.0/token", - ) - resource_uri = get_m365d_endpoint(cloud) - api_version = "api" - api_endpoint = "/advancedhunting/run" else: + if data_environment == DataEnvironment.M365D: + logger.warning( + "M365 Defender/Defender XDR Advanced Hunting API has been deprecated." + "Reverting to MDE Advanced Queries API." + "Please use Microsoft Graph Security Hunting API instead - " + "provider name = 'M365DGraph'." + ) + + # MDE Advanced Queries API logger.info("Using MDE Advanced Queries API (default)") login_uri = urljoin( get_m365d_login_endpoint(cloud), diff --git a/msticpy/data/drivers/odata_driver.py b/msticpy/data/drivers/odata_driver.py index bcf50d09..d4aa1b6d 100644 --- a/msticpy/data/drivers/odata_driver.py +++ b/msticpy/data/drivers/odata_driver.py @@ -558,7 +558,11 @@ def _get_driver_settings( logger.debug( "Getting driver settings for: %s (instance: %s)", config_name, instance ) - config_key: str = f"{config_name}-{instance}" if instance else config_name + config_key: str = ( + f"{config_name}-{instance}" + if instance and instance != "Default" + else config_name + ) drv_config: ProviderSettings | None = get_provider_settings("DataProviders").get( config_key, ) diff --git a/tests/data/drivers/test_mdatp_driver.py b/tests/data/drivers/test_mdatp_driver.py index a3b11809..5bce85a7 100644 --- a/tests/data/drivers/test_mdatp_driver.py +++ b/tests/data/drivers/test_mdatp_driver.py @@ -50,9 +50,10 @@ def test_select_api_mde() -> None: def test_select_api_m365d() -> None: """Test API selection for M365 Defender unified environment.""" + # Note this now reverts to MDE parameters cfg = _select_api(DataEnvironment.M365D, "global") - assert cfg.api_endpoint == "/advancedhunting/run" - assert "/api/advancedhunting/run" in cfg.api_uri + assert cfg.api_endpoint == "/advancedqueries/run" + assert "/advancedqueries/run" in cfg.api_uri def test_select_api_graph() -> None: diff --git a/tests/data/drivers/test_odata_drivers.py b/tests/data/drivers/test_odata_drivers.py index a4a110c2..61c4c5a9 100644 --- a/tests/data/drivers/test_odata_drivers.py +++ b/tests/data/drivers/test_odata_drivers.py @@ -45,10 +45,10 @@ _MDEF_TESTS = [ ("MDE", "https://api.securitycenter.microsoft.com/", ""), ("MDATP", "https://api.securitycenter.microsoft.com/", None), - ("M365D", "https://api.security.microsoft.com/", None), - ("M365D", "https://api-us.security.microsoft.com/", "us"), - ("M365D", "https://api-eu.security.microsoft.com/", "eu"), - ("M365D", "https://api-uk.security.microsoft.com/", "uk"), + ("M365D", "https://api.securitycenter.microsoft.com/", None), + ("M365D", "https://api-us.securitycenter.microsoft.com/", "us"), + ("M365D", "https://api-eu.securitycenter.microsoft.com/", "eu"), + ("M365D", "https://api-uk.securitycenter.microsoft.com/", "uk"), ("MDE", "https://api-gov.securitycenter.microsoft.us/", "dod"), ("MDE", "https://api-uk.securitycenter.microsoft.com/", "uk"), ("MDE", "https://api-us.securitycenter.microsoft.com/", "us"), @@ -108,7 +108,7 @@ def _mde_create_mock(httpx): ("MDATP", "securitycenter"), ("MDE", "securitycenter"), ("MDE", "securitycenter"), - ("M365D", "security"), + ("M365D", "securitycenter"), ], ) @patch("msticpy.data.drivers.odata_driver.httpx") @@ -128,7 +128,7 @@ def test_mde_connect(httpx, env, api): _MDE_CONNECT_STR = [ ("MDATP", "securitycenter", _CONSTRING), ("MDE", "securitycenter", _CONSTRING), - ("M365D", "security", _CONSTRING), + ("M365D", "securitycenter", _CONSTRING), ] @@ -154,7 +154,7 @@ def test_mde_connect_str(httpx, env, api, con_str): _MDE_CONNECT_PARAMS = [ ("MDATP", "securitycenter", _PARAMS), ("MDE", "securitycenter", _PARAMS), - ("M365D", "security", _PARAMS), + ("M365D", "securitycenter", _PARAMS), ] From 1bb76cef1eb8bd27893fb6cf3c4e6c11a14df321 Mon Sep 17 00:00:00 2001 From: ianhelle Date: Tue, 21 Oct 2025 13:32:47 -0700 Subject: [PATCH 2/8] Fixing pylint errors from new pylint version Fixing aiagents # Conflicts: # conda/conda-reqs-pip.txt # requirements-all.txt # setup.py --- .pylintrc | 4 ---- msticpy/common/provider_settings.py | 1 + msticpy/context/azure/azure_data.py | 4 ++-- msticpy/context/ip_utils.py | 1 + msticpy/context/tiproviders/riskiq.py | 4 ++-- msticpy/context/vtlookupv3/__init__.py | 2 +- msticpy/context/vtlookupv3/vtfile_behavior.py | 4 ++-- msticpy/context/vtlookupv3/vtlookupv3.py | 4 ++-- msticpy/data/drivers/azure_kusto_driver.py | 4 ++-- msticpy/data/drivers/mordor_driver.py | 1 + msticpy/data/sql_to_kql.py | 1 + msticpy/datamodel/entities/entity_enums.py | 2 +- msticpy/init/nbinit.py | 17 +++++++------- msticpy/transform/base64unpack.py | 22 ++++++++++--------- 14 files changed, 37 insertions(+), 34 deletions(-) diff --git a/.pylintrc b/.pylintrc index b7db2e9e..4e5473e8 100644 --- a/.pylintrc +++ b/.pylintrc @@ -36,10 +36,6 @@ persistent=yes # Specify a configuration file. #rcfile= -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode=yes - # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no diff --git a/msticpy/common/provider_settings.py b/msticpy/common/provider_settings.py index d087f4ee..e8382c0b 100644 --- a/msticpy/common/provider_settings.py +++ b/msticpy/common/provider_settings.py @@ -98,6 +98,7 @@ def _return_secrets_client( # Create a SecretsClient instance if it can be imported when # the module is imported. +# pylint: disable=invalid-name _SECRETS_CLIENT: Any = None # Create the secrets client closure _SET_SECRETS_CLIENT: Callable[..., "SecretsClient" | None] = get_secrets_client_func() diff --git a/msticpy/context/azure/azure_data.py b/msticpy/context/azure/azure_data.py index b9735271..29375b70 100644 --- a/msticpy/context/azure/azure_data.py +++ b/msticpy/context/azure/azure_data.py @@ -54,7 +54,7 @@ from azure.mgmt.network.models import NetworkInterface from azure.mgmt.subscription.models import Subscription except ImportError as imp_err: - error_msg: str = ( + ERROR_MSG: str = ( "Cannot use this feature without these azure packages installed:\n" "azure.mgmt.network\n" "azure.mgmt.resource\n" @@ -62,7 +62,7 @@ "azure.mgmt.compute\n" ) raise MsticpyImportExtraError( - error_msg, + ERROR_MSG, title="Error importing azure module", extra="azure", ) from imp_err diff --git a/msticpy/context/ip_utils.py b/msticpy/context/ip_utils.py index 32f860cd..41ef1e89 100644 --- a/msticpy/context/ip_utils.py +++ b/msticpy/context/ip_utils.py @@ -91,6 +91,7 @@ def _get_asns_dict() -> dict[str, str]: # Create the dictionary accessor from the fetch_asns wrapper +# pylint: disable=invalid-name _ASNS_DICT: Callable[[], dict[str, str]] = _fetch_asns() diff --git a/msticpy/context/tiproviders/riskiq.py b/msticpy/context/tiproviders/riskiq.py index a51eea06..5fed85bf 100644 --- a/msticpy/context/tiproviders/riskiq.py +++ b/msticpy/context/tiproviders/riskiq.py @@ -38,9 +38,9 @@ from passivetotal.analyzer.whois import WhoisRecords except ImportError as imp_err: - error_msg: str = "Cannot use this feature without passivetotal package installed." + ERROR_MSG: str = "Cannot use this feature without passivetotal package installed." raise MsticpyImportExtraError( - error_msg, + ERROR_MSG, title="Error importing RiskIQ modules.", extra="riskiq", ) from imp_err diff --git a/msticpy/context/vtlookupv3/__init__.py b/msticpy/context/vtlookupv3/__init__.py index 1cb77e8e..ecb82d33 100644 --- a/msticpy/context/vtlookupv3/__init__.py +++ b/msticpy/context/vtlookupv3/__init__.py @@ -8,7 +8,7 @@ from ..._version import VERSION -# pylint: disable=unused-import +# pylint: disable=unused-import, invalid-name # flake8: noqa: F401 VT3_AVAILABLE = False with contextlib.suppress(ImportError): diff --git a/msticpy/context/vtlookupv3/vtfile_behavior.py b/msticpy/context/vtlookupv3/vtfile_behavior.py index 4d969483..6d10b192 100644 --- a/msticpy/context/vtlookupv3/vtfile_behavior.py +++ b/msticpy/context/vtlookupv3/vtfile_behavior.py @@ -30,11 +30,11 @@ try: import vt except ImportError as imp_err: - err_msg: str = ( + ERR_MSG: str = ( "Cannot use this feature without vt-py and vt-graph-api packages installed." ) raise MsticpyImportExtraError( - err_msg, + ERR_MSG, title="Error importing VirusTotal modules.", extra="vt3", ) from imp_err diff --git a/msticpy/context/vtlookupv3/vtlookupv3.py b/msticpy/context/vtlookupv3/vtlookupv3.py index b6041bf6..abdc5c25 100644 --- a/msticpy/context/vtlookupv3/vtlookupv3.py +++ b/msticpy/context/vtlookupv3/vtlookupv3.py @@ -32,12 +32,12 @@ from vt.object import Object except ImportError as imp_err: - err_msg: str = ( + ERR_MSG: str = ( "Cannot use this feature without vt-py, vt-graph-api and " "nest_asyncio packages installed." ) raise MsticpyImportExtraError( - err_msg, + ERR_MSG, title="Error importing VirusTotal modules.", extra="vt3", ) from imp_err diff --git a/msticpy/data/drivers/azure_kusto_driver.py b/msticpy/data/drivers/azure_kusto_driver.py index 541e5061..76de95d4 100644 --- a/msticpy/data/drivers/azure_kusto_driver.py +++ b/msticpy/data/drivers/azure_kusto_driver.py @@ -48,9 +48,9 @@ if TYPE_CHECKING: from azure.kusto.data.response import KustoResponseDataSet except ImportError as imp_err: - import_err: str = "Cannot use this feature without Azure Kusto client installed" + IMPORT_ERR: str = "Cannot use this feature without Azure Kusto client installed" raise MsticpyMissingDependencyError( - import_err, + IMPORT_ERR, title="Error importing azure.kusto.data", packages="azure-kusto-data", ) from imp_err diff --git a/msticpy/data/drivers/mordor_driver.py b/msticpy/data/drivers/mordor_driver.py index b4402953..062da78d 100644 --- a/msticpy/data/drivers/mordor_driver.py +++ b/msticpy/data/drivers/mordor_driver.py @@ -41,6 +41,7 @@ _MTR_TAC_CAT_URI = "https://attack.mitre.org/tactics/{cat}/" _MTR_TECH_CAT_URI = "https://attack.mitre.org/techniques/{cat}/" +# pylint: disable=invalid-name MITRE_TECHNIQUES: Optional[pd.DataFrame] = None MITRE_TACTICS: Optional[pd.DataFrame] = None diff --git a/msticpy/data/sql_to_kql.py b/msticpy/data/sql_to_kql.py index 6f871458..2a2cd6c8 100644 --- a/msticpy/data/sql_to_kql.py +++ b/msticpy/data/sql_to_kql.py @@ -143,6 +143,7 @@ WHERE = "where" WITH = "with" +# pylint: disable=invalid-name JOIN_KEYWORDS = { FULL_JOIN: "outer", FULL_OUTER_JOIN: "outer", diff --git a/msticpy/datamodel/entities/entity_enums.py b/msticpy/datamodel/entities/entity_enums.py index 36c3baa3..e625ea99 100644 --- a/msticpy/datamodel/entities/entity_enums.py +++ b/msticpy/datamodel/entities/entity_enums.py @@ -13,7 +13,7 @@ __version__ = VERSION __author__ = "Ian Hellen" - +# pylint: disable=invalid-name ENTITY_ENUMS: Dict[str, Type] = {} diff --git a/msticpy/init/nbinit.py b/msticpy/init/nbinit.py index 0ca0c769..1b425987 100644 --- a/msticpy/init/nbinit.py +++ b/msticpy/init/nbinit.py @@ -192,7 +192,8 @@ def _verbose(verbosity: int | None = None) -> int: return _verbose -_VERBOSITY: Callable[[int | None], int] = _get_verbosity_setting() +# pylint: disable=invalid-name +VERBOSITY: Callable[[int | None], int] = _get_verbosity_setting() # pylint: disable=use-dict-literal _NB_IMPORTS = [ @@ -442,7 +443,7 @@ def init_notebook( if friendly_exceptions is None: friendly_exceptions = get_config("msticpy.FriendlyExceptions", None) if friendly_exceptions: - if _VERBOSITY() == 2: # type: ignore + if VERBOSITY() == 2: # type: ignore _pr_output("Friendly exceptions enabled.") InteractiveShell.showtraceback = _hook_ipython_exceptions( # type: ignore InteractiveShell.showtraceback, @@ -462,7 +463,7 @@ def init_notebook( def _pr_output(*args): """Output to IPython display or print.""" - if not _VERBOSITY(): + if not VERBOSITY(): return if is_ipython(): display(HTML(" ".join([*args, "
"]).replace("\n", "
"))) @@ -581,7 +582,7 @@ def _set_verbosity(**kwargs): verbosity = 2 if verb_param else 0 elif isinstance(verb_param, int): verbosity = min(2, max(0, verb_param)) - _VERBOSITY(verbosity) + VERBOSITY(verbosity) def _detect_env(env_name: Literal["aml", "synapse"], **kwargs): @@ -854,7 +855,7 @@ def _imp_module(nm_spc: dict[str, Any], module_name: str, alias: str | None = No nm_spc[alias] = mod else: nm_spc[module_name] = mod - if _VERBOSITY() == 2: # type: ignore + if VERBOSITY() == 2: # type: ignore _pr_output(f"{module_name} imported (alias={alias})") return mod @@ -870,7 +871,7 @@ def _imp_module_all(nm_spc: dict[str, Any], module_name): if item.startswith("_"): continue nm_spc[item] = getattr(imported_mod, item) - if _VERBOSITY() == 2: # type: ignore + if VERBOSITY() == 2: # type: ignore _pr_output(f"All items imported from {module_name}") @@ -898,7 +899,7 @@ def _imp_from_package( nm_spc[alias] = obj else: nm_spc[tgt] = obj - if _VERBOSITY() == 2: # type: ignore + if VERBOSITY() == 2: # type: ignore _pr_output(f"{tgt} imported from {pkg} (alias={alias})") return obj @@ -931,7 +932,7 @@ def _check_and_reload_pkg( importlib.reload(pkg) else: _imp_module(nm_spc, pkg_name, alias=alias) - if _VERBOSITY() == 2: # type: ignore + if VERBOSITY() == 2: # type: ignore _pr_output(f"{pkg_name} imported version {pkg.__version__}") return warn_mssg diff --git a/msticpy/transform/base64unpack.py b/msticpy/transform/base64unpack.py index 4eef588a..11d64ff8 100644 --- a/msticpy/transform/base64unpack.py +++ b/msticpy/transform/base64unpack.py @@ -116,7 +116,8 @@ def _trace_enabled(trace: Optional[bool] = None) -> bool: return _trace_enabled -_GET_TRACE = _get_trace_setting() +# pylint: disable=invalid-name +GET_TRACE = _get_trace_setting() def _get_utf16_setting() -> Callable[[Optional[bool]], bool]: @@ -132,7 +133,8 @@ def _utf16_enabled(utf16: Optional[bool] = None) -> bool: return _utf16_enabled -_GET_UTF16 = _get_utf16_setting() +# pylint: disable=invalid-name +GET_UTF16 = _get_utf16_setting() @export @@ -203,8 +205,8 @@ def unpack_items( frame. This allows you to re-join the output data to the input data. """ - _GET_TRACE(trace) - _GET_UTF16(utf16) + GET_TRACE(trace) + GET_UTF16(utf16) if input_string is not None: input_string = _b64_string_pad(input_string) @@ -252,8 +254,8 @@ def unpack( replaced by the results of the decoding """ - _GET_TRACE(trace) - _GET_UTF16(utf16) + GET_TRACE(trace) + GET_UTF16(utf16) return _decode_b64_string_recursive(input_string) @@ -312,8 +314,8 @@ def unpack_df( frame. """ - _GET_TRACE(trace) - _GET_UTF16(utf16) + GET_TRACE(trace) + GET_UTF16(utf16) output_df = pd.DataFrame(columns=BinaryRecord._fields) row_results: List[pd.DataFrame] = [] @@ -460,7 +462,7 @@ def _add_to_results( def _debug_print_trace(*args): - if _GET_TRACE(): + if GET_TRACE(): for arg in args: print(arg, end="") print() @@ -587,7 +589,7 @@ def _get_byte_encoding(bytes_array: bytes) -> BinaryRecord: """ result_rec = _empty_binary_rec() printable_bytes = _as_byte_string(bytes_array) - if _GET_UTF16(): # type: ignore + if GET_UTF16(): # type: ignore try: # Difficult to tell the difference between a real unicode string # and a binary string that happens to decode to a utf-16 string. From bebb54fac14b7b3b6ad0ec5b86af0284eb17e604 Mon Sep 17 00:00:00 2001 From: ianhelle Date: Tue, 21 Oct 2025 13:34:41 -0700 Subject: [PATCH 3/8] Adding space to warning message --- msticpy/data/drivers/mdatp_driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msticpy/data/drivers/mdatp_driver.py b/msticpy/data/drivers/mdatp_driver.py index 03e93113..9500b0df 100644 --- a/msticpy/data/drivers/mdatp_driver.py +++ b/msticpy/data/drivers/mdatp_driver.py @@ -290,7 +290,7 @@ def _select_api(data_environment: DataEnvironment, cloud: str) -> M365DConfigura if data_environment == DataEnvironment.M365D: logger.warning( "M365 Defender/Defender XDR Advanced Hunting API has been deprecated." - "Reverting to MDE Advanced Queries API." + "Reverting to MDE Advanced Queries API. " "Please use Microsoft Graph Security Hunting API instead - " "provider name = 'M365DGraph'." ) From d2be821f8da082745747ed1eb5aef413bd9a8196 Mon Sep 17 00:00:00 2001 From: ianhelle Date: Tue, 21 Oct 2025 13:41:02 -0700 Subject: [PATCH 4/8] Updating .gitignore to exclude Guardian analysis files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 9b001704..6c9df62f 100644 --- a/.gitignore +++ b/.gitignore @@ -120,3 +120,6 @@ morphchart_package/ /docs/notebooks/kqlmagic/* /kqlmagic/** /GitExtensions.settings + +# Guardian files +/.gdn/** From 0e3859aec783e0eb6f0ff11c2f9fe599271df650 Mon Sep 17 00:00:00 2001 From: ianhelle Date: Tue, 21 Oct 2025 14:27:39 -0700 Subject: [PATCH 5/8] Fixing pylint errors --- .pre-commit-config.yaml | 10 +++++----- msticpy/context/tiproviders/pulsedive.py | 14 +++++++------- msticpy/data/drivers/kql_driver.py | 1 - 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e957ab5c..928742cd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,19 +8,19 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - repo: https://github.com/ambv/black - rev: 24.10.0 + rev: 25.9.0 hooks: - id: black language: python - repo: https://github.com/PyCQA/pylint - rev: v3.3.1 + rev: v4.0.2 hooks: - id: pylint args: - --disable=duplicate-code,import-error - --ignore-patterns=test_ - repo: https://github.com/pycqa/flake8 - rev: 7.1.1 + rev: 7.3.0 hooks: - id: flake8 args: @@ -28,7 +28,7 @@ repos: - --max-line-length=90 - --exclude=tests,test*.py - repo: https://github.com/pycqa/isort - rev: 5.13.2 + rev: 7.0.0 hooks: - id: isort name: isort (python) @@ -43,7 +43,7 @@ repos: - --convention=numpy - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.8.0 + rev: v0.14.1 hooks: # Run the linter. - id: ruff diff --git a/msticpy/context/tiproviders/pulsedive.py b/msticpy/context/tiproviders/pulsedive.py index 0d32a16b..dab0422f 100644 --- a/msticpy/context/tiproviders/pulsedive.py +++ b/msticpy/context/tiproviders/pulsedive.py @@ -30,8 +30,6 @@ __version__ = VERSION __author__ = "Thomas Roccia | @fr0gger_" -_BASE_URL = "https://pulsedive.com/api/" - _QUERY_OBJECTS_MAPPINGS: dict[str, dict[str, str]] = { "indicator": {"indicator": "observable"}, "threat": {"threat": "observable"}, @@ -76,6 +74,8 @@ class PDlookup: """ + BASE_URL = "https://pulsedive.com/api/" + _SUPPORTED_PD_TYPES: ClassVar[set[PDEntityType]] = { PDEntityType.INDICATOR, PDEntityType.THREAT, @@ -230,7 +230,7 @@ def _make_pd_request(self: Self, pd_query: PDQuery) -> pd.DataFrame: **mp_ua_header(), } resp: httpx.Response = httpx.post( - f"{_BASE_URL}analyze.php", + f"{self.BASE_URL}analyze.php", data=data, headers=headers, ) @@ -244,9 +244,9 @@ def _make_pd_request(self: Self, pd_query: PDQuery) -> pd.DataFrame: if pd_query.query_type == "q": # if the key is "q", make a GET request to the explore endpoint - query_url: str = f"{_BASE_URL}explore.php" + query_url: str = f"{self.BASE_URL}explore.php" else: - query_url = f"{_BASE_URL}info.php" + query_url = f"{self.BASE_URL}info.php" params: dict[str, Any] = { **self._get_default_params(), pd_query.query_type: pd_query.data, @@ -280,7 +280,7 @@ def _poll_for_results(self: Self, resp: httpx.Response) -> pd.DataFrame: params: dict[str, Any] = {"qid": qid, **self._get_default_params()} headers: dict[str, str] = mp_ua_header() timeout: httpx.Timeout = get_http_timeout() - url: str = f"{_BASE_URL}analyze.php" + url: str = f"{self.BASE_URL}analyze.php" resp = httpx.get( url, params=params, @@ -344,7 +344,7 @@ def _build_query_string(data: str, pd_type: str) -> PDQuery: class Pulsedive(HttpTIProvider): """Pulsedive TI Lookup.""" - _BASE_URL = _BASE_URL + _BASE_URL = PDlookup.BASE_URL _QUERIES: ClassVar[dict[str, APILookupParams]] = { ioc_type: _QUERY_DEF for ioc_type in ("ipv4", "ipv6", "dns", "hostname", "url") diff --git a/msticpy/data/drivers/kql_driver.py b/msticpy/data/drivers/kql_driver.py index 9a7e318b..97aa5988 100644 --- a/msticpy/data/drivers/kql_driver.py +++ b/msticpy/data/drivers/kql_driver.py @@ -74,7 +74,6 @@ def _set_kql_env_option(option, value): _KQL_CLOUD_MAP = {"global": "public", "cn": "china", "usgov": "government"} _KQL_OPTIONS = ["timeout"] -_KQL_ENV_OPTS = "KQLMAGIC_CONFIGURATION" _AZ_CLOUD_MAP = {kql_cloud: az_cloud for az_cloud, kql_cloud in _KQL_CLOUD_MAP.items()} From ff6c9b253675a8b88f664ec2ec1087bfe11222cd Mon Sep 17 00:00:00 2001 From: ianhelle Date: Wed, 22 Oct 2025 14:10:10 -0700 Subject: [PATCH 6/8] Fixing some M365D queries --- .../data/queries/m365d/kql_m365_alerts.yaml | 41 +++++-------------- 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/msticpy/data/queries/m365d/kql_m365_alerts.yaml b/msticpy/data/queries/m365d/kql_m365_alerts.yaml index 77b08476..048ddebb 100644 --- a/msticpy/data/queries/m365d/kql_m365_alerts.yaml +++ b/msticpy/data/queries/m365d/kql_m365_alerts.yaml @@ -44,9 +44,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 {add_query_items}" uri: None host_alerts: @@ -81,9 +79,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType =~ "Ip" | where RemoteIP has "{ip_address}" {add_query_items}' @@ -102,9 +98,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType =~ "Url" | where RemoteUrl has "{url}" {add_query_items}' @@ -123,9 +117,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType in~ ("File", "Process") | where SHA1 has "{file_hash}" {add_query_items}' @@ -146,9 +138,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType in~ ("File", "Process") | where SHA256 has "{file_hash}" {add_query_items}' @@ -169,9 +159,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType =~ "Process" | where FileName =~ "{file_name}" {add_query_items}' @@ -192,9 +180,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType =~ "User" | where AccountName =~ "{account_name}" {add_query_items}' @@ -213,9 +199,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType =~ "RegistryValue" | where RegistryKey =~ "{key_name}" {add_query_items}' @@ -234,9 +218,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType =~ "MailMessage" | where NetworkMessageId =~ "{message_id}" {add_query_items}' @@ -255,9 +237,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType =~ "Mailbox" | where AccountUpn =~ "{mailbox}" {add_query_items}' @@ -276,6 +256,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId + | project-away *1 | where EntityType in~ ("CloudApplication", "OAuthApplication") | where Application =~ "{app_name}" {add_query_items}' From 9f949aec96605e5c9fda292a8b7670ee9c4238d6 Mon Sep 17 00:00:00 2001 From: ianhelle Date: Wed, 29 Oct 2025 10:25:41 -0700 Subject: [PATCH 7/8] Changing logger.warning to deprecation warning (added logging.captureWarnings so that this also goes to logs) --- msticpy/data/drivers/mdatp_driver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/msticpy/data/drivers/mdatp_driver.py b/msticpy/data/drivers/mdatp_driver.py index 9500b0df..1ad17538 100644 --- a/msticpy/data/drivers/mdatp_driver.py +++ b/msticpy/data/drivers/mdatp_driver.py @@ -7,6 +7,7 @@ from __future__ import annotations import logging +import warnings from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, ClassVar from urllib.parse import urljoin @@ -28,6 +29,7 @@ __version__ = VERSION __author__ = "Pete Bryan" +logging.captureWarnings(True) logger: logging.Logger = logging.getLogger(__name__) @@ -288,12 +290,13 @@ def _select_api(data_environment: DataEnvironment, cloud: str) -> M365DConfigura api_endpoint = "/security/runHuntingQuery" else: if data_environment == DataEnvironment.M365D: - logger.warning( + warn_message = ( "M365 Defender/Defender XDR Advanced Hunting API has been deprecated." "Reverting to MDE Advanced Queries API. " "Please use Microsoft Graph Security Hunting API instead - " "provider name = 'M365DGraph'." ) + warnings.warn(warn_message, DeprecationWarning) # MDE Advanced Queries API logger.info("Using MDE Advanced Queries API (default)") From 1e3b5acd04ad5cf246f81c4eccebe286356cd5fe Mon Sep 17 00:00:00 2001 From: ianhelle Date: Fri, 31 Oct 2025 13:53:18 -0700 Subject: [PATCH 8/8] Updating Defender documentation --- .../data_acquisition/DataProv-MSDefender.rst | 163 +++++++++++------- 1 file changed, 100 insertions(+), 63 deletions(-) diff --git a/docs/source/data_acquisition/DataProv-MSDefender.rst b/docs/source/data_acquisition/DataProv-MSDefender.rst index 520c32b6..5087ae86 100644 --- a/docs/source/data_acquisition/DataProv-MSDefender.rst +++ b/docs/source/data_acquisition/DataProv-MSDefender.rst @@ -1,28 +1,35 @@ -Microsoft 365 Defender Provider -=============================== +Microsoft Defender Provider +=========================== This driver lets you query the Microsoft Defender APIs. -.. note:: This section applies to both Microsoft 365 Defender and Microsoft Defender - for Endpoint (MDE). The former supersedes and is a subset of the the latter - but both are still available to use. +.. note:: This provider supports multiple API endpoints for accessing Microsoft Defender data: + + - **M365DGraph** (Recommended): Uses the Microsoft Graph API with ThreatHunting permissions. + This is the preferred and most modern API. + - **MDE/MDATP**: Uses the Microsoft Defender for Endpoint API (formerly MDATP). + This is the fallback option when M365DGraph is not available. + - **M365D** (Removed): The Microsoft 365 Defender API provider has been removed from + MSTICPy. If you use the "M365D" provider name, it will automatically use the MDE + API endpoints instead. Many components in MSTICPy still use the old abbreviation **MDATP** - (Microsoft Advanced Threat Protection). + (Microsoft Advanced Threat Protection) for backwards compatibility. -M365 Defender Configuration ---------------------------- +Defender Configuration +---------------------- -Creating a Client App for M365 Defender -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Creating a Client App for Microsoft Defender +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Microsoft 365 Defender APIs can be accessed in both `application ` -and `delegated user contexts `. -Accessing Microsoft 365 Defender APIs as an application requires +Microsoft Defender APIs can be accessed in both +`application `__ +and `delegated user contexts `__. +Accessing Microsoft Defender APIs as an application requires either a client secret or certificate, while delegated user auth requires an interactive signin through a browser or via device code. -As such, the details on registering an Azure AD application for MS 365 Defender +As such, the details on registering an Azure AD application for MS Defender are different for application and delegated user auth scenarios. Please see the above links for more information. Notably, delegated user auth scenarios do not require a application credential and thus is preferrable. @@ -38,18 +45,24 @@ and auth scenario (application or delegated user): +-----------------------------+------------------------+------------------+ | API Name | Permission | Data Environment | +=============================+========================+==================+ -| WindowsDefenderATP | AdvancedQuery.Read | MDE, MDATP | -+-----------------------------+------------------------+------------------+ -| Microsoft Threat Protection | AdvancedHunting.Read | M365D | -+-----------------------------+------------------------+------------------+ | Microsoft Graph | ThreatHunting.Read.All | M365DGraph | +-----------------------------+------------------------+------------------+ +| WindowsDefenderATP | AdvancedQuery.Read | MDE, MDATP | ++-----------------------------+------------------------+------------------+ + +.. note:: **M365DGraph is the recommended data environment** as it uses the modern + Microsoft Graph API. The MDE/MDATP endpoints are maintained for backwards + compatibility and as a fallback option. + + The Microsoft Threat Protection API provider (M365D) has been removed. If you + specify "M365D" as the provider name, it will automatically use MDE endpoints + for backwards compatibility. Once you have registered the application, you can use it to connect to -the MS Defender API using the chosen data environment. +the MS Defender API using your chosen data environment. -M365 Defender Configuration in MSTICPy -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Defender Configuration in MSTICPy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can store your connection details in *msticpyconfig.yaml*. @@ -88,7 +101,7 @@ Your configuration when using Key Vault should look like the following: KeyVault: TenantId: "TENANT ID" -You can create multiple instances of M365 Defender settings by adding +You can create multiple instances of Defender settings by adding an instance string to the "MicrosoftDefender" section name. .. code:: yaml @@ -134,55 +147,79 @@ See :doc:`msticpy Settings Editor <../getting_started/SettingsEditor>`. PrivateKeySecret: KeyVault: -Loading a QueryProvider for M365 Defender ------------------------------------------ +Loading a QueryProvider for Microsoft Defender +---------------------------------------------- + +**Recommended: Use M365DGraph** + +.. code:: ipython3 + + defender_prov = QueryProvider("M365DGraph") + +**Alternative: Use MDE (legacy/fallback)** .. code:: ipython3 - mdatp_prov = QueryProvider("M365D") + mde_prov = QueryProvider("MDE") -You can also use the aliases "MDE" and "MDATP". +You can also use the alias "MDATP" for backwards compatibility. -Specifying the Defender Cloud Instance to Connect to ----------------------------------------------------- +.. note:: The "M365D" alias has been removed but is still accepted for backwards + compatibility - it will automatically use MDE endpoints. We recommend using + "M365DGraph" for new implementations. + +Specifying the Defender Cloud Instance +-------------------------------------- If connecting to the Defender API to run queries there are a number of different endpoints you can connect to. Which one is most applicable will depend on your location and which cloud you are using. -By default 'https://api.securitycenter.microsoft.com/' or -'https://api.security.microsoft.com/' is used, but others can be -specified either in your MSTICPy config file, or by passing -in the name with the cloud keyword: +**For M365DGraph (Microsoft Graph API):** + +By default 'https://graph.microsoft.com/' is used. For government clouds, +the appropriate Graph endpoint will be selected automatically based on the +cloud parameter. .. code:: ipython3 - mdatp_prov = QueryProvider("MDE", cloud="gcc") + defender_prov = QueryProvider("M365DGraph", cloud="gcc") +**For MDE (Defender for Endpoint API):** -If using an MDE-specific API endpoint, the "name" (the first parameter to QueryProvider in the example above) must be "MDE". +By default 'https://api.securitycenter.microsoft.com/' is used, but others can be +specified either in your MSTICPy config file, or by passing +in the name with the cloud keyword: -+----------+----------------------------------------------+----------------------------------------+ -| Cloud | MDE | M365D | -+==========+==============================================+========================================+ -| global | https://api.securitycenter.microsoft.com/ | https://api.security.microsoft.com/ | -+----------+----------------------------------------------+----------------------------------------+ -| uk | https://api-uk.securitycenter.microsoft.com/ | https://api-uk.security.microsoft.com/ | -+----------+----------------------------------------------+----------------------------------------+ -| us | https://api-us.securitycenter.microsoft.com/ | https://api-us.security.microsoft.com/ | -+----------+----------------------------------------------+----------------------------------------+ -| eu | https://api-eu.securitycenter.microsoft.com/ | https://api-eu.security.microsoft.com/ | -+----------+----------------------------------------------+----------------------------------------+ -| gcc | https://api-gcc.securitycenter.microsoft.us/ | NA | -+----------+----------------------------------------------+----------------------------------------+ -| gcc-high | https://api-gov.securitycenter.microsoft.us/ | NA | -+----------+----------------------------------------------+----------------------------------------+ -| dod | https://api-gov.securitycenter.microsoft.us/ | NA | -+----------+----------------------------------------------+----------------------------------------+ +.. code:: ipython3 -Connecting to M365 Defender ---------------------------- + mde_prov = QueryProvider("MDE", cloud="gcc") + + ++----------+----------------------------------------------+ +| Cloud | MDE Endpoint | ++==========+==============================================+ +| global | https://api.securitycenter.microsoft.com/ | ++----------+----------------------------------------------+ +| uk | https://api-uk.securitycenter.microsoft.com/ | ++----------+----------------------------------------------+ +| us | https://api-us.securitycenter.microsoft.com/ | ++----------+----------------------------------------------+ +| eu | https://api-eu.securitycenter.microsoft.com/ | ++----------+----------------------------------------------+ +| gcc | https://api-gcc.securitycenter.microsoft.us/ | ++----------+----------------------------------------------+ +| gcc-high | https://api-gov.securitycenter.microsoft.us/ | ++----------+----------------------------------------------+ +| dod | https://api-gov.securitycenter.microsoft.us/ | ++----------+----------------------------------------------+ + +.. note:: M365DGraph uses Microsoft Graph endpoints which are automatically + configured for government clouds. The above table applies only to MDE/MDATP endpoints. + +Connecting to Microsoft Defender +-------------------------------- The parameters required for connection to Defender can be passed in a number of ways. The simplest is to configure your settings @@ -190,7 +227,7 @@ in msticpyconfig. You can then just call connect with no parameters. .. code:: ipython3 - mdatp_prov.connect() + defender_prov.connect() If you have configured multiple instances you must specify @@ -198,7 +235,7 @@ an instance name when you call connect. .. code:: ipython3 - mdatp_prov.connect(instance="Tenant2") + defender_prov.connect(instance="Tenant2") If you want to use delegated authentication for your application you can specify this when you call connect. By default, this will @@ -208,7 +245,7 @@ auth_type to "device". .. code:: ipython3 - mdatp_prov.connect(delegated_auth=True, auth_type="device") + defender_prov.connect(delegated_auth=True, auth_type="device") You can also pass connection parameters as keyword arguments or a connection string. @@ -228,8 +265,8 @@ The client_secret and username parameters are mutually exclusive. ten_id = input('Tenant ID') client_id = input('Client ID') client_secret = input('Client Secret') - md_prov = QueryProvider('M365D') - md_prov.connect(tenant_id=ten_id, client_id=client_id, client_secret=client_secret) + defender_prov = QueryProvider('M365DGraph') + defender_prov.connect(tenant_id=ten_id, client_id=client_id, client_secret=client_secret) You can also specify these parameters as a connection string of the form: @@ -246,12 +283,12 @@ You can also specify these parameters as a connection string of the form: ) md_prov.connect(conn_str) -Other M365 Defender Documentation ---------------------------------- +Other Microsoft Defender Documentation +-------------------------------------- For examples of using the MS Defender provider, see the sample -`M365 Defender Notebook` +`Microsoft Defender Notebook`__ -Built-in :ref:`data_acquisition/DataQueries:Queries for Microsoft 365 Defender`. +Built-in :ref:`data_acquisition/DataQueries:Queries for Microsoft Defender`. -:py:mod:`M365 Defender driver API documentation` +:py:mod:`Microsoft Defender driver API documentation`