diff --git a/.gitignore b/.gitignore index 9b0017040..6c9df62f3 100644 --- a/.gitignore +++ b/.gitignore @@ -120,3 +120,6 @@ morphchart_package/ /docs/notebooks/kqlmagic/* /kqlmagic/** /GitExtensions.settings + +# Guardian files +/.gdn/** diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e957ab5c3..928742cd9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,19 +8,19 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - repo: https://github.com/ambv/black - rev: 24.10.0 + rev: 25.9.0 hooks: - id: black language: python - repo: https://github.com/PyCQA/pylint - rev: v3.3.1 + rev: v4.0.2 hooks: - id: pylint args: - --disable=duplicate-code,import-error - --ignore-patterns=test_ - repo: https://github.com/pycqa/flake8 - rev: 7.1.1 + rev: 7.3.0 hooks: - id: flake8 args: @@ -28,7 +28,7 @@ repos: - --max-line-length=90 - --exclude=tests,test*.py - repo: https://github.com/pycqa/isort - rev: 5.13.2 + rev: 7.0.0 hooks: - id: isort name: isort (python) @@ -43,7 +43,7 @@ repos: - --convention=numpy - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.8.0 + rev: v0.14.1 hooks: # Run the linter. - id: ruff diff --git a/.pylintrc b/.pylintrc index b7db2e9ec..4e5473e8e 100644 --- a/.pylintrc +++ b/.pylintrc @@ -36,10 +36,6 @@ persistent=yes # Specify a configuration file. #rcfile= -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode=yes - # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no diff --git a/docs/source/data_acquisition/DataProv-MSDefender.rst b/docs/source/data_acquisition/DataProv-MSDefender.rst index 520c32b69..5087ae860 100644 --- a/docs/source/data_acquisition/DataProv-MSDefender.rst +++ b/docs/source/data_acquisition/DataProv-MSDefender.rst @@ -1,28 +1,35 @@ -Microsoft 365 Defender Provider -=============================== +Microsoft Defender Provider +=========================== This driver lets you query the Microsoft Defender APIs. -.. note:: This section applies to both Microsoft 365 Defender and Microsoft Defender - for Endpoint (MDE). The former supersedes and is a subset of the the latter - but both are still available to use. +.. note:: This provider supports multiple API endpoints for accessing Microsoft Defender data: + + - **M365DGraph** (Recommended): Uses the Microsoft Graph API with ThreatHunting permissions. + This is the preferred and most modern API. + - **MDE/MDATP**: Uses the Microsoft Defender for Endpoint API (formerly MDATP). + This is the fallback option when M365DGraph is not available. + - **M365D** (Removed): The Microsoft 365 Defender API provider has been removed from + MSTICPy. If you use the "M365D" provider name, it will automatically use the MDE + API endpoints instead. Many components in MSTICPy still use the old abbreviation **MDATP** - (Microsoft Advanced Threat Protection). + (Microsoft Advanced Threat Protection) for backwards compatibility. -M365 Defender Configuration ---------------------------- +Defender Configuration +---------------------- -Creating a Client App for M365 Defender -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Creating a Client App for Microsoft Defender +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Microsoft 365 Defender APIs can be accessed in both `application ` -and `delegated user contexts `. -Accessing Microsoft 365 Defender APIs as an application requires +Microsoft Defender APIs can be accessed in both +`application `__ +and `delegated user contexts `__. +Accessing Microsoft Defender APIs as an application requires either a client secret or certificate, while delegated user auth requires an interactive signin through a browser or via device code. -As such, the details on registering an Azure AD application for MS 365 Defender +As such, the details on registering an Azure AD application for MS Defender are different for application and delegated user auth scenarios. Please see the above links for more information. Notably, delegated user auth scenarios do not require a application credential and thus is preferrable. @@ -38,18 +45,24 @@ and auth scenario (application or delegated user): +-----------------------------+------------------------+------------------+ | API Name | Permission | Data Environment | +=============================+========================+==================+ -| WindowsDefenderATP | AdvancedQuery.Read | MDE, MDATP | -+-----------------------------+------------------------+------------------+ -| Microsoft Threat Protection | AdvancedHunting.Read | M365D | -+-----------------------------+------------------------+------------------+ | Microsoft Graph | ThreatHunting.Read.All | M365DGraph | +-----------------------------+------------------------+------------------+ +| WindowsDefenderATP | AdvancedQuery.Read | MDE, MDATP | ++-----------------------------+------------------------+------------------+ + +.. note:: **M365DGraph is the recommended data environment** as it uses the modern + Microsoft Graph API. The MDE/MDATP endpoints are maintained for backwards + compatibility and as a fallback option. + + The Microsoft Threat Protection API provider (M365D) has been removed. If you + specify "M365D" as the provider name, it will automatically use MDE endpoints + for backwards compatibility. Once you have registered the application, you can use it to connect to -the MS Defender API using the chosen data environment. +the MS Defender API using your chosen data environment. -M365 Defender Configuration in MSTICPy -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Defender Configuration in MSTICPy +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can store your connection details in *msticpyconfig.yaml*. @@ -88,7 +101,7 @@ Your configuration when using Key Vault should look like the following: KeyVault: TenantId: "TENANT ID" -You can create multiple instances of M365 Defender settings by adding +You can create multiple instances of Defender settings by adding an instance string to the "MicrosoftDefender" section name. .. code:: yaml @@ -134,55 +147,79 @@ See :doc:`msticpy Settings Editor <../getting_started/SettingsEditor>`. PrivateKeySecret: KeyVault: -Loading a QueryProvider for M365 Defender ------------------------------------------ +Loading a QueryProvider for Microsoft Defender +---------------------------------------------- + +**Recommended: Use M365DGraph** + +.. code:: ipython3 + + defender_prov = QueryProvider("M365DGraph") + +**Alternative: Use MDE (legacy/fallback)** .. code:: ipython3 - mdatp_prov = QueryProvider("M365D") + mde_prov = QueryProvider("MDE") -You can also use the aliases "MDE" and "MDATP". +You can also use the alias "MDATP" for backwards compatibility. -Specifying the Defender Cloud Instance to Connect to ----------------------------------------------------- +.. note:: The "M365D" alias has been removed but is still accepted for backwards + compatibility - it will automatically use MDE endpoints. We recommend using + "M365DGraph" for new implementations. + +Specifying the Defender Cloud Instance +-------------------------------------- If connecting to the Defender API to run queries there are a number of different endpoints you can connect to. Which one is most applicable will depend on your location and which cloud you are using. -By default 'https://api.securitycenter.microsoft.com/' or -'https://api.security.microsoft.com/' is used, but others can be -specified either in your MSTICPy config file, or by passing -in the name with the cloud keyword: +**For M365DGraph (Microsoft Graph API):** + +By default 'https://graph.microsoft.com/' is used. For government clouds, +the appropriate Graph endpoint will be selected automatically based on the +cloud parameter. .. code:: ipython3 - mdatp_prov = QueryProvider("MDE", cloud="gcc") + defender_prov = QueryProvider("M365DGraph", cloud="gcc") +**For MDE (Defender for Endpoint API):** -If using an MDE-specific API endpoint, the "name" (the first parameter to QueryProvider in the example above) must be "MDE". +By default 'https://api.securitycenter.microsoft.com/' is used, but others can be +specified either in your MSTICPy config file, or by passing +in the name with the cloud keyword: -+----------+----------------------------------------------+----------------------------------------+ -| Cloud | MDE | M365D | -+==========+==============================================+========================================+ -| global | https://api.securitycenter.microsoft.com/ | https://api.security.microsoft.com/ | -+----------+----------------------------------------------+----------------------------------------+ -| uk | https://api-uk.securitycenter.microsoft.com/ | https://api-uk.security.microsoft.com/ | -+----------+----------------------------------------------+----------------------------------------+ -| us | https://api-us.securitycenter.microsoft.com/ | https://api-us.security.microsoft.com/ | -+----------+----------------------------------------------+----------------------------------------+ -| eu | https://api-eu.securitycenter.microsoft.com/ | https://api-eu.security.microsoft.com/ | -+----------+----------------------------------------------+----------------------------------------+ -| gcc | https://api-gcc.securitycenter.microsoft.us/ | NA | -+----------+----------------------------------------------+----------------------------------------+ -| gcc-high | https://api-gov.securitycenter.microsoft.us/ | NA | -+----------+----------------------------------------------+----------------------------------------+ -| dod | https://api-gov.securitycenter.microsoft.us/ | NA | -+----------+----------------------------------------------+----------------------------------------+ +.. code:: ipython3 -Connecting to M365 Defender ---------------------------- + mde_prov = QueryProvider("MDE", cloud="gcc") + + ++----------+----------------------------------------------+ +| Cloud | MDE Endpoint | ++==========+==============================================+ +| global | https://api.securitycenter.microsoft.com/ | ++----------+----------------------------------------------+ +| uk | https://api-uk.securitycenter.microsoft.com/ | ++----------+----------------------------------------------+ +| us | https://api-us.securitycenter.microsoft.com/ | ++----------+----------------------------------------------+ +| eu | https://api-eu.securitycenter.microsoft.com/ | ++----------+----------------------------------------------+ +| gcc | https://api-gcc.securitycenter.microsoft.us/ | ++----------+----------------------------------------------+ +| gcc-high | https://api-gov.securitycenter.microsoft.us/ | ++----------+----------------------------------------------+ +| dod | https://api-gov.securitycenter.microsoft.us/ | ++----------+----------------------------------------------+ + +.. note:: M365DGraph uses Microsoft Graph endpoints which are automatically + configured for government clouds. The above table applies only to MDE/MDATP endpoints. + +Connecting to Microsoft Defender +-------------------------------- The parameters required for connection to Defender can be passed in a number of ways. The simplest is to configure your settings @@ -190,7 +227,7 @@ in msticpyconfig. You can then just call connect with no parameters. .. code:: ipython3 - mdatp_prov.connect() + defender_prov.connect() If you have configured multiple instances you must specify @@ -198,7 +235,7 @@ an instance name when you call connect. .. code:: ipython3 - mdatp_prov.connect(instance="Tenant2") + defender_prov.connect(instance="Tenant2") If you want to use delegated authentication for your application you can specify this when you call connect. By default, this will @@ -208,7 +245,7 @@ auth_type to "device". .. code:: ipython3 - mdatp_prov.connect(delegated_auth=True, auth_type="device") + defender_prov.connect(delegated_auth=True, auth_type="device") You can also pass connection parameters as keyword arguments or a connection string. @@ -228,8 +265,8 @@ The client_secret and username parameters are mutually exclusive. ten_id = input('Tenant ID') client_id = input('Client ID') client_secret = input('Client Secret') - md_prov = QueryProvider('M365D') - md_prov.connect(tenant_id=ten_id, client_id=client_id, client_secret=client_secret) + defender_prov = QueryProvider('M365DGraph') + defender_prov.connect(tenant_id=ten_id, client_id=client_id, client_secret=client_secret) You can also specify these parameters as a connection string of the form: @@ -246,12 +283,12 @@ You can also specify these parameters as a connection string of the form: ) md_prov.connect(conn_str) -Other M365 Defender Documentation ---------------------------------- +Other Microsoft Defender Documentation +-------------------------------------- For examples of using the MS Defender provider, see the sample -`M365 Defender Notebook` +`Microsoft Defender Notebook`__ -Built-in :ref:`data_acquisition/DataQueries:Queries for Microsoft 365 Defender`. +Built-in :ref:`data_acquisition/DataQueries:Queries for Microsoft Defender`. -:py:mod:`M365 Defender driver API documentation` +:py:mod:`Microsoft Defender driver API documentation` diff --git a/msticpy/_version.py b/msticpy/_version.py index 3ec49d689..bf65a81fb 100644 --- a/msticpy/_version.py +++ b/msticpy/_version.py @@ -1,3 +1,3 @@ """Version file.""" -VERSION = "2.17.1" +VERSION = "2.17.2" diff --git a/msticpy/common/provider_settings.py b/msticpy/common/provider_settings.py index d087f4ee3..e8382c0bf 100644 --- a/msticpy/common/provider_settings.py +++ b/msticpy/common/provider_settings.py @@ -98,6 +98,7 @@ def _return_secrets_client( # Create a SecretsClient instance if it can be imported when # the module is imported. +# pylint: disable=invalid-name _SECRETS_CLIENT: Any = None # Create the secrets client closure _SET_SECRETS_CLIENT: Callable[..., "SecretsClient" | None] = get_secrets_client_func() diff --git a/msticpy/context/azure/azure_data.py b/msticpy/context/azure/azure_data.py index b9735271f..29375b707 100644 --- a/msticpy/context/azure/azure_data.py +++ b/msticpy/context/azure/azure_data.py @@ -54,7 +54,7 @@ from azure.mgmt.network.models import NetworkInterface from azure.mgmt.subscription.models import Subscription except ImportError as imp_err: - error_msg: str = ( + ERROR_MSG: str = ( "Cannot use this feature without these azure packages installed:\n" "azure.mgmt.network\n" "azure.mgmt.resource\n" @@ -62,7 +62,7 @@ "azure.mgmt.compute\n" ) raise MsticpyImportExtraError( - error_msg, + ERROR_MSG, title="Error importing azure module", extra="azure", ) from imp_err diff --git a/msticpy/context/ip_utils.py b/msticpy/context/ip_utils.py index 32f860cdd..41ef1e892 100644 --- a/msticpy/context/ip_utils.py +++ b/msticpy/context/ip_utils.py @@ -91,6 +91,7 @@ def _get_asns_dict() -> dict[str, str]: # Create the dictionary accessor from the fetch_asns wrapper +# pylint: disable=invalid-name _ASNS_DICT: Callable[[], dict[str, str]] = _fetch_asns() diff --git a/msticpy/context/tiproviders/pulsedive.py b/msticpy/context/tiproviders/pulsedive.py index 0d32a16bc..dab0422f1 100644 --- a/msticpy/context/tiproviders/pulsedive.py +++ b/msticpy/context/tiproviders/pulsedive.py @@ -30,8 +30,6 @@ __version__ = VERSION __author__ = "Thomas Roccia | @fr0gger_" -_BASE_URL = "https://pulsedive.com/api/" - _QUERY_OBJECTS_MAPPINGS: dict[str, dict[str, str]] = { "indicator": {"indicator": "observable"}, "threat": {"threat": "observable"}, @@ -76,6 +74,8 @@ class PDlookup: """ + BASE_URL = "https://pulsedive.com/api/" + _SUPPORTED_PD_TYPES: ClassVar[set[PDEntityType]] = { PDEntityType.INDICATOR, PDEntityType.THREAT, @@ -230,7 +230,7 @@ def _make_pd_request(self: Self, pd_query: PDQuery) -> pd.DataFrame: **mp_ua_header(), } resp: httpx.Response = httpx.post( - f"{_BASE_URL}analyze.php", + f"{self.BASE_URL}analyze.php", data=data, headers=headers, ) @@ -244,9 +244,9 @@ def _make_pd_request(self: Self, pd_query: PDQuery) -> pd.DataFrame: if pd_query.query_type == "q": # if the key is "q", make a GET request to the explore endpoint - query_url: str = f"{_BASE_URL}explore.php" + query_url: str = f"{self.BASE_URL}explore.php" else: - query_url = f"{_BASE_URL}info.php" + query_url = f"{self.BASE_URL}info.php" params: dict[str, Any] = { **self._get_default_params(), pd_query.query_type: pd_query.data, @@ -280,7 +280,7 @@ def _poll_for_results(self: Self, resp: httpx.Response) -> pd.DataFrame: params: dict[str, Any] = {"qid": qid, **self._get_default_params()} headers: dict[str, str] = mp_ua_header() timeout: httpx.Timeout = get_http_timeout() - url: str = f"{_BASE_URL}analyze.php" + url: str = f"{self.BASE_URL}analyze.php" resp = httpx.get( url, params=params, @@ -344,7 +344,7 @@ def _build_query_string(data: str, pd_type: str) -> PDQuery: class Pulsedive(HttpTIProvider): """Pulsedive TI Lookup.""" - _BASE_URL = _BASE_URL + _BASE_URL = PDlookup.BASE_URL _QUERIES: ClassVar[dict[str, APILookupParams]] = { ioc_type: _QUERY_DEF for ioc_type in ("ipv4", "ipv6", "dns", "hostname", "url") diff --git a/msticpy/context/tiproviders/riskiq.py b/msticpy/context/tiproviders/riskiq.py index a51eea064..5fed85bfc 100644 --- a/msticpy/context/tiproviders/riskiq.py +++ b/msticpy/context/tiproviders/riskiq.py @@ -38,9 +38,9 @@ from passivetotal.analyzer.whois import WhoisRecords except ImportError as imp_err: - error_msg: str = "Cannot use this feature without passivetotal package installed." + ERROR_MSG: str = "Cannot use this feature without passivetotal package installed." raise MsticpyImportExtraError( - error_msg, + ERROR_MSG, title="Error importing RiskIQ modules.", extra="riskiq", ) from imp_err diff --git a/msticpy/context/vtlookupv3/__init__.py b/msticpy/context/vtlookupv3/__init__.py index 1cb77e8e8..ecb82d330 100644 --- a/msticpy/context/vtlookupv3/__init__.py +++ b/msticpy/context/vtlookupv3/__init__.py @@ -8,7 +8,7 @@ from ..._version import VERSION -# pylint: disable=unused-import +# pylint: disable=unused-import, invalid-name # flake8: noqa: F401 VT3_AVAILABLE = False with contextlib.suppress(ImportError): diff --git a/msticpy/context/vtlookupv3/vtfile_behavior.py b/msticpy/context/vtlookupv3/vtfile_behavior.py index 4d9694839..6d10b1924 100644 --- a/msticpy/context/vtlookupv3/vtfile_behavior.py +++ b/msticpy/context/vtlookupv3/vtfile_behavior.py @@ -30,11 +30,11 @@ try: import vt except ImportError as imp_err: - err_msg: str = ( + ERR_MSG: str = ( "Cannot use this feature without vt-py and vt-graph-api packages installed." ) raise MsticpyImportExtraError( - err_msg, + ERR_MSG, title="Error importing VirusTotal modules.", extra="vt3", ) from imp_err diff --git a/msticpy/context/vtlookupv3/vtlookupv3.py b/msticpy/context/vtlookupv3/vtlookupv3.py index b6041bf6b..abdc5c255 100644 --- a/msticpy/context/vtlookupv3/vtlookupv3.py +++ b/msticpy/context/vtlookupv3/vtlookupv3.py @@ -32,12 +32,12 @@ from vt.object import Object except ImportError as imp_err: - err_msg: str = ( + ERR_MSG: str = ( "Cannot use this feature without vt-py, vt-graph-api and " "nest_asyncio packages installed." ) raise MsticpyImportExtraError( - err_msg, + ERR_MSG, title="Error importing VirusTotal modules.", extra="vt3", ) from imp_err diff --git a/msticpy/data/core/query_source.py b/msticpy/data/core/query_source.py index a8da146d3..f62641928 100644 --- a/msticpy/data/core/query_source.py +++ b/msticpy/data/core/query_source.py @@ -27,7 +27,7 @@ def _value_or_default(src_dict: dict, prop_name: str, default: dict): - """Return value from dict or emtpy dict.""" + """Return value from dict or empty dict.""" src_value = src_dict.get(prop_name) return src_value if src_value is not None else default diff --git a/msticpy/data/drivers/azure_kusto_driver.py b/msticpy/data/drivers/azure_kusto_driver.py index 541e50615..76de95d44 100644 --- a/msticpy/data/drivers/azure_kusto_driver.py +++ b/msticpy/data/drivers/azure_kusto_driver.py @@ -48,9 +48,9 @@ if TYPE_CHECKING: from azure.kusto.data.response import KustoResponseDataSet except ImportError as imp_err: - import_err: str = "Cannot use this feature without Azure Kusto client installed" + IMPORT_ERR: str = "Cannot use this feature without Azure Kusto client installed" raise MsticpyMissingDependencyError( - import_err, + IMPORT_ERR, title="Error importing azure.kusto.data", packages="azure-kusto-data", ) from imp_err diff --git a/msticpy/data/drivers/azure_monitor_driver.py b/msticpy/data/drivers/azure_monitor_driver.py index 9b13f8f09..984d3d738 100644 --- a/msticpy/data/drivers/azure_monitor_driver.py +++ b/msticpy/data/drivers/azure_monitor_driver.py @@ -19,7 +19,6 @@ import contextlib import logging import warnings -from datetime import datetime from typing import Any, Iterable, cast import httpx @@ -112,7 +111,7 @@ def __init__(self, connection_str: str | None = None, **kwargs): self._schema: dict[str, Any] = {} self.set_driver_property( DriverProps.FORMATTERS, - {"datetime": self._format_datetime, "list": self._format_list}, + {"list": self._format_list}, ) self._loaded = True self._ua_policy = UserAgentPolicy(user_agent=mp_ua_header()["UserAgent"]) @@ -639,11 +638,6 @@ def _get_schema(self) -> dict[str, dict]: ) return _schema_format_tables(tables) - @staticmethod - def _format_datetime(date_time: datetime) -> str: - """Return datetime-formatted string.""" - return date_time.isoformat(sep="T") + "Z" - @staticmethod def _format_list(param_list: Iterable[Any]): """Return formatted list parameter.""" diff --git a/msticpy/data/drivers/kql_driver.py b/msticpy/data/drivers/kql_driver.py index 9a7e318b5..97aa5988c 100644 --- a/msticpy/data/drivers/kql_driver.py +++ b/msticpy/data/drivers/kql_driver.py @@ -74,7 +74,6 @@ def _set_kql_env_option(option, value): _KQL_CLOUD_MAP = {"global": "public", "cn": "china", "usgov": "government"} _KQL_OPTIONS = ["timeout"] -_KQL_ENV_OPTS = "KQLMAGIC_CONFIGURATION" _AZ_CLOUD_MAP = {kql_cloud: az_cloud for az_cloud, kql_cloud in _KQL_CLOUD_MAP.items()} diff --git a/msticpy/data/drivers/mdatp_driver.py b/msticpy/data/drivers/mdatp_driver.py index 0cd2ac4f2..1ad175386 100644 --- a/msticpy/data/drivers/mdatp_driver.py +++ b/msticpy/data/drivers/mdatp_driver.py @@ -7,6 +7,7 @@ from __future__ import annotations import logging +import warnings from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, ClassVar from urllib.parse import urljoin @@ -16,11 +17,7 @@ from ..._version import VERSION from ...auth.azure_auth_core import AzureCloudConfig -from ...auth.cloud_mappings import ( - get_defender_endpoint, - get_m365d_endpoint, - get_m365d_login_endpoint, -) +from ...auth.cloud_mappings import get_defender_endpoint, get_m365d_login_endpoint from ...common.data_utils import ensure_df_datetimes from ...common.utility import export from ..core.query_defns import DataEnvironment @@ -32,6 +29,7 @@ __version__ = VERSION __author__ = "Pete Bryan" +logging.captureWarnings(True) logger: logging.Logger = logging.getLogger(__name__) @@ -117,7 +115,7 @@ def __init__( self.cloud: str = cs_dict.pop("cloud", "global") if cloud: - logger.info("Overriding configured cloud with: %s", cloud) + logger.info("Using Azure cloud: %s", cloud) self.cloud = cloud else: logger.debug("Using cloud from configuration: %s", self.cloud) @@ -290,16 +288,17 @@ def _select_api(data_environment: DataEnvironment, cloud: str) -> M365DConfigura resource_uri: str = az_cloud_config.endpoints["microsoftGraphResourceId"] api_version = "v1.0" api_endpoint = "/security/runHuntingQuery" - elif data_environment == DataEnvironment.M365D: - logger.info("Using M365 Defender Advanced Hunting API") - login_uri = urljoin( - get_m365d_login_endpoint(cloud), - "{tenantId}/oauth2/v2.0/token", - ) - resource_uri = get_m365d_endpoint(cloud) - api_version = "api" - api_endpoint = "/advancedhunting/run" else: + if data_environment == DataEnvironment.M365D: + warn_message = ( + "M365 Defender/Defender XDR Advanced Hunting API has been deprecated." + "Reverting to MDE Advanced Queries API. " + "Please use Microsoft Graph Security Hunting API instead - " + "provider name = 'M365DGraph'." + ) + warnings.warn(warn_message, DeprecationWarning) + + # MDE Advanced Queries API logger.info("Using MDE Advanced Queries API (default)") login_uri = urljoin( get_m365d_login_endpoint(cloud), diff --git a/msticpy/data/drivers/mordor_driver.py b/msticpy/data/drivers/mordor_driver.py index b4402953a..062da78d1 100644 --- a/msticpy/data/drivers/mordor_driver.py +++ b/msticpy/data/drivers/mordor_driver.py @@ -41,6 +41,7 @@ _MTR_TAC_CAT_URI = "https://attack.mitre.org/tactics/{cat}/" _MTR_TECH_CAT_URI = "https://attack.mitre.org/techniques/{cat}/" +# pylint: disable=invalid-name MITRE_TECHNIQUES: Optional[pd.DataFrame] = None MITRE_TACTICS: Optional[pd.DataFrame] = None diff --git a/msticpy/data/drivers/odata_driver.py b/msticpy/data/drivers/odata_driver.py index bcf50d099..d4aa1b6da 100644 --- a/msticpy/data/drivers/odata_driver.py +++ b/msticpy/data/drivers/odata_driver.py @@ -558,7 +558,11 @@ def _get_driver_settings( logger.debug( "Getting driver settings for: %s (instance: %s)", config_name, instance ) - config_key: str = f"{config_name}-{instance}" if instance else config_name + config_key: str = ( + f"{config_name}-{instance}" + if instance and instance != "Default" + else config_name + ) drv_config: ProviderSettings | None = get_provider_settings("DataProviders").get( config_key, ) diff --git a/msticpy/data/queries/m365d/kql_m365_alerts.yaml b/msticpy/data/queries/m365d/kql_m365_alerts.yaml index 77b084760..048ddebb8 100644 --- a/msticpy/data/queries/m365d/kql_m365_alerts.yaml +++ b/msticpy/data/queries/m365d/kql_m365_alerts.yaml @@ -44,9 +44,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 {add_query_items}" uri: None host_alerts: @@ -81,9 +79,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType =~ "Ip" | where RemoteIP has "{ip_address}" {add_query_items}' @@ -102,9 +98,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType =~ "Url" | where RemoteUrl has "{url}" {add_query_items}' @@ -123,9 +117,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType in~ ("File", "Process") | where SHA1 has "{file_hash}" {add_query_items}' @@ -146,9 +138,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType in~ ("File", "Process") | where SHA256 has "{file_hash}" {add_query_items}' @@ -169,9 +159,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType =~ "Process" | where FileName =~ "{file_name}" {add_query_items}' @@ -192,9 +180,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType =~ "User" | where AccountName =~ "{account_name}" {add_query_items}' @@ -213,9 +199,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType =~ "RegistryValue" | where RegistryKey =~ "{key_name}" {add_query_items}' @@ -234,9 +218,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType =~ "MailMessage" | where NetworkMessageId =~ "{message_id}" {add_query_items}' @@ -255,9 +237,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId - | project-away TenantId1, TimeGenerated1, Timestamp1, AlertId1, Title1, - AttackTechniques1, ServiceSource1, DetectionSource1, SHA1, - SourceSystem1, Type1 + | project-away *1 | where EntityType =~ "Mailbox" | where AccountUpn =~ "{mailbox}" {add_query_items}' @@ -276,6 +256,7 @@ sources: | join kind=inner (AlertEvidence | where Timestamp >= datetime({start}) | where Timestamp <= datetime({end})) on AlertId + | project-away *1 | where EntityType in~ ("CloudApplication", "OAuthApplication") | where Application =~ "{app_name}" {add_query_items}' diff --git a/msticpy/data/sql_to_kql.py b/msticpy/data/sql_to_kql.py index 6f871458c..2a2cd6c80 100644 --- a/msticpy/data/sql_to_kql.py +++ b/msticpy/data/sql_to_kql.py @@ -143,6 +143,7 @@ WHERE = "where" WITH = "with" +# pylint: disable=invalid-name JOIN_KEYWORDS = { FULL_JOIN: "outer", FULL_OUTER_JOIN: "outer", diff --git a/msticpy/datamodel/entities/entity_enums.py b/msticpy/datamodel/entities/entity_enums.py index 36c3baa30..e625ea991 100644 --- a/msticpy/datamodel/entities/entity_enums.py +++ b/msticpy/datamodel/entities/entity_enums.py @@ -13,7 +13,7 @@ __version__ = VERSION __author__ = "Ian Hellen" - +# pylint: disable=invalid-name ENTITY_ENUMS: Dict[str, Type] = {} diff --git a/msticpy/init/nbinit.py b/msticpy/init/nbinit.py index 0ca0c7699..1b4259878 100644 --- a/msticpy/init/nbinit.py +++ b/msticpy/init/nbinit.py @@ -192,7 +192,8 @@ def _verbose(verbosity: int | None = None) -> int: return _verbose -_VERBOSITY: Callable[[int | None], int] = _get_verbosity_setting() +# pylint: disable=invalid-name +VERBOSITY: Callable[[int | None], int] = _get_verbosity_setting() # pylint: disable=use-dict-literal _NB_IMPORTS = [ @@ -442,7 +443,7 @@ def init_notebook( if friendly_exceptions is None: friendly_exceptions = get_config("msticpy.FriendlyExceptions", None) if friendly_exceptions: - if _VERBOSITY() == 2: # type: ignore + if VERBOSITY() == 2: # type: ignore _pr_output("Friendly exceptions enabled.") InteractiveShell.showtraceback = _hook_ipython_exceptions( # type: ignore InteractiveShell.showtraceback, @@ -462,7 +463,7 @@ def init_notebook( def _pr_output(*args): """Output to IPython display or print.""" - if not _VERBOSITY(): + if not VERBOSITY(): return if is_ipython(): display(HTML(" ".join([*args, "
"]).replace("\n", "
"))) @@ -581,7 +582,7 @@ def _set_verbosity(**kwargs): verbosity = 2 if verb_param else 0 elif isinstance(verb_param, int): verbosity = min(2, max(0, verb_param)) - _VERBOSITY(verbosity) + VERBOSITY(verbosity) def _detect_env(env_name: Literal["aml", "synapse"], **kwargs): @@ -854,7 +855,7 @@ def _imp_module(nm_spc: dict[str, Any], module_name: str, alias: str | None = No nm_spc[alias] = mod else: nm_spc[module_name] = mod - if _VERBOSITY() == 2: # type: ignore + if VERBOSITY() == 2: # type: ignore _pr_output(f"{module_name} imported (alias={alias})") return mod @@ -870,7 +871,7 @@ def _imp_module_all(nm_spc: dict[str, Any], module_name): if item.startswith("_"): continue nm_spc[item] = getattr(imported_mod, item) - if _VERBOSITY() == 2: # type: ignore + if VERBOSITY() == 2: # type: ignore _pr_output(f"All items imported from {module_name}") @@ -898,7 +899,7 @@ def _imp_from_package( nm_spc[alias] = obj else: nm_spc[tgt] = obj - if _VERBOSITY() == 2: # type: ignore + if VERBOSITY() == 2: # type: ignore _pr_output(f"{tgt} imported from {pkg} (alias={alias})") return obj @@ -931,7 +932,7 @@ def _check_and_reload_pkg( importlib.reload(pkg) else: _imp_module(nm_spc, pkg_name, alias=alias) - if _VERBOSITY() == 2: # type: ignore + if VERBOSITY() == 2: # type: ignore _pr_output(f"{pkg_name} imported version {pkg.__version__}") return warn_mssg diff --git a/msticpy/transform/base64unpack.py b/msticpy/transform/base64unpack.py index 4eef588a4..11d64ff8f 100644 --- a/msticpy/transform/base64unpack.py +++ b/msticpy/transform/base64unpack.py @@ -116,7 +116,8 @@ def _trace_enabled(trace: Optional[bool] = None) -> bool: return _trace_enabled -_GET_TRACE = _get_trace_setting() +# pylint: disable=invalid-name +GET_TRACE = _get_trace_setting() def _get_utf16_setting() -> Callable[[Optional[bool]], bool]: @@ -132,7 +133,8 @@ def _utf16_enabled(utf16: Optional[bool] = None) -> bool: return _utf16_enabled -_GET_UTF16 = _get_utf16_setting() +# pylint: disable=invalid-name +GET_UTF16 = _get_utf16_setting() @export @@ -203,8 +205,8 @@ def unpack_items( frame. This allows you to re-join the output data to the input data. """ - _GET_TRACE(trace) - _GET_UTF16(utf16) + GET_TRACE(trace) + GET_UTF16(utf16) if input_string is not None: input_string = _b64_string_pad(input_string) @@ -252,8 +254,8 @@ def unpack( replaced by the results of the decoding """ - _GET_TRACE(trace) - _GET_UTF16(utf16) + GET_TRACE(trace) + GET_UTF16(utf16) return _decode_b64_string_recursive(input_string) @@ -312,8 +314,8 @@ def unpack_df( frame. """ - _GET_TRACE(trace) - _GET_UTF16(utf16) + GET_TRACE(trace) + GET_UTF16(utf16) output_df = pd.DataFrame(columns=BinaryRecord._fields) row_results: List[pd.DataFrame] = [] @@ -460,7 +462,7 @@ def _add_to_results( def _debug_print_trace(*args): - if _GET_TRACE(): + if GET_TRACE(): for arg in args: print(arg, end="") print() @@ -587,7 +589,7 @@ def _get_byte_encoding(bytes_array: bytes) -> BinaryRecord: """ result_rec = _empty_binary_rec() printable_bytes = _as_byte_string(bytes_array) - if _GET_UTF16(): # type: ignore + if GET_UTF16(): # type: ignore try: # Difficult to tell the difference between a real unicode string # and a binary string that happens to decode to a utf-16 string. diff --git a/tests/data/drivers/test_mdatp_driver.py b/tests/data/drivers/test_mdatp_driver.py index a3b11809b..5bce85a7d 100644 --- a/tests/data/drivers/test_mdatp_driver.py +++ b/tests/data/drivers/test_mdatp_driver.py @@ -50,9 +50,10 @@ def test_select_api_mde() -> None: def test_select_api_m365d() -> None: """Test API selection for M365 Defender unified environment.""" + # Note this now reverts to MDE parameters cfg = _select_api(DataEnvironment.M365D, "global") - assert cfg.api_endpoint == "/advancedhunting/run" - assert "/api/advancedhunting/run" in cfg.api_uri + assert cfg.api_endpoint == "/advancedqueries/run" + assert "/advancedqueries/run" in cfg.api_uri def test_select_api_graph() -> None: diff --git a/tests/data/drivers/test_odata_drivers.py b/tests/data/drivers/test_odata_drivers.py index a4a110c24..61c4c5a90 100644 --- a/tests/data/drivers/test_odata_drivers.py +++ b/tests/data/drivers/test_odata_drivers.py @@ -45,10 +45,10 @@ _MDEF_TESTS = [ ("MDE", "https://api.securitycenter.microsoft.com/", ""), ("MDATP", "https://api.securitycenter.microsoft.com/", None), - ("M365D", "https://api.security.microsoft.com/", None), - ("M365D", "https://api-us.security.microsoft.com/", "us"), - ("M365D", "https://api-eu.security.microsoft.com/", "eu"), - ("M365D", "https://api-uk.security.microsoft.com/", "uk"), + ("M365D", "https://api.securitycenter.microsoft.com/", None), + ("M365D", "https://api-us.securitycenter.microsoft.com/", "us"), + ("M365D", "https://api-eu.securitycenter.microsoft.com/", "eu"), + ("M365D", "https://api-uk.securitycenter.microsoft.com/", "uk"), ("MDE", "https://api-gov.securitycenter.microsoft.us/", "dod"), ("MDE", "https://api-uk.securitycenter.microsoft.com/", "uk"), ("MDE", "https://api-us.securitycenter.microsoft.com/", "us"), @@ -108,7 +108,7 @@ def _mde_create_mock(httpx): ("MDATP", "securitycenter"), ("MDE", "securitycenter"), ("MDE", "securitycenter"), - ("M365D", "security"), + ("M365D", "securitycenter"), ], ) @patch("msticpy.data.drivers.odata_driver.httpx") @@ -128,7 +128,7 @@ def test_mde_connect(httpx, env, api): _MDE_CONNECT_STR = [ ("MDATP", "securitycenter", _CONSTRING), ("MDE", "securitycenter", _CONSTRING), - ("M365D", "security", _CONSTRING), + ("M365D", "securitycenter", _CONSTRING), ] @@ -154,7 +154,7 @@ def test_mde_connect_str(httpx, env, api, con_str): _MDE_CONNECT_PARAMS = [ ("MDATP", "securitycenter", _PARAMS), ("MDE", "securitycenter", _PARAMS), - ("M365D", "security", _PARAMS), + ("M365D", "securitycenter", _PARAMS), ]