From 49c07b6176730c7ee454986e694b9419ca51de71 Mon Sep 17 00:00:00 2001 From: "Meroujan.Antonyan" Date: Fri, 18 Oct 2024 11:46:13 +0000 Subject: [PATCH 1/8] Add max retry parameter to the execution --- msticpy/data/drivers/cybereason_driver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/msticpy/data/drivers/cybereason_driver.py b/msticpy/data/drivers/cybereason_driver.py index 3901f7f1f..2750b010d 100644 --- a/msticpy/data/drivers/cybereason_driver.py +++ b/msticpy/data/drivers/cybereason_driver.py @@ -401,6 +401,7 @@ def __execute_query( page: int = 0, page_size: int = 2000, pagination_token: str = None, + max_retry: int = 3, ) -> Dict[str, Any]: """ Run query with pagination enabled. @@ -436,7 +437,8 @@ def __execute_query( headers = {} params = {"page": page, "itemsPerPage": page_size} status = None - while status != "SUCCESS": + cur_try = 0 + while status != "SUCCESS" and cur_try < max_retry: response = self.client.post( self.search_endpoint, json={**body, **pagination}, @@ -446,6 +448,7 @@ def __execute_query( response.raise_for_status() json_result = response.json() status = json_result["status"] + cur_try += 1 return json_result async def __run_threaded_queries( From 8c7278381e53e7ad0afac0bb8509be419ffbcc6b Mon Sep 17 00:00:00 2001 From: "Meroujan.Antonyan" Date: Tue, 22 Oct 2024 08:32:46 +0000 Subject: [PATCH 2/8] Add tests and raise exception after max retry --- msticpy/data/drivers/cybereason_driver.py | 8 +++ tests/data/drivers/test_cybereason_driver.py | 66 ++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/msticpy/data/drivers/cybereason_driver.py b/msticpy/data/drivers/cybereason_driver.py index 2750b010d..05edc2f15 100644 --- a/msticpy/data/drivers/cybereason_driver.py +++ b/msticpy/data/drivers/cybereason_driver.py @@ -449,6 +449,14 @@ def __execute_query( json_result = response.json() status = json_result["status"] cur_try += 1 + + if cur_try >= max_retry: + raise httpx.HTTPStatusError( + f"{status}: {json_result['message']}", + request=response.request, + response=response + ) + return json_result async def __run_threaded_queries( diff --git a/tests/data/drivers/test_cybereason_driver.py b/tests/data/drivers/test_cybereason_driver.py index 0dc8f8d66..317b91aad 100644 --- a/tests/data/drivers/test_cybereason_driver.py +++ b/tests/data/drivers/test_cybereason_driver.py @@ -12,6 +12,7 @@ import pytest import pytest_check as check import respx +import httpx from msticpy.data.core.query_defns import Formatters from msticpy.data.drivers.cybereason_driver import CybereasonDriver @@ -140,6 +141,46 @@ }, ] +_CR_PARTIAL_SUCCESS_RESULT = { + "data": { + "resultIdToElementDataMap": {}, + "suspicionsMap": {}, + "evidenceMap": {}, + "totalResults": 0, + "totalPossibleResults": 0, + "guessedPossibleResults": 0, + "queryLimits": { + "totalResultLimit": 1000, + "perGroupLimit": 100, + "perFeatureLimit": 100, + "groupingFeature": { + "elementInstanceType": "Process", + "featureName": "imageFileHash", + }, + "sortInGroupFeature": None, + }, + "queryTerminated": False, + "pathResultCounts": None, + "guids": [], + "paginationToken": None, + "executionUUID": None, + "quapiMeasurementData": { + "timeToGetGuids": [], + "timeToGetData": [], + "timeToGetAdditionalData": [], + "totalQuapiQueryTime": [], + "startTime": [], + "endTime": [], + }, + }, + "status": "PARTIAL_SUCCESS", + "hidePartialSuccess": False, + "message": "Received Non-OK status code HTTP/1.1 500 Internal Server Error", + "expectedResults": 0, + "failures": 0, + "failedServersInfo": None, +} + _CR_QUERY = { "query": """ { @@ -241,6 +282,31 @@ def test_query(driver): check.is_true(query.called) check.is_instance(data, pd.DataFrame) +@respx.mock +def test_partial_success_query(driver): + """Test query calling returns data in expected format.""" + connect = respx.post( + re.compile(r"^https://[a-zA-Z0-9\-]+\.cybereason\.net/login\.html") + ).respond(200) + query = respx.post( + re.compile( + r"^https://[a-zA-Z0-9\-]+\.cybereason\.net/rest/visualsearch/query/simple" + ) + ) + query.side_effect = [ + httpx.Response(200, json=_CR_PARTIAL_SUCCESS_RESULT), + httpx.Response(200, json=_CR_PARTIAL_SUCCESS_RESULT), + httpx.Response(200, json=_CR_PARTIAL_SUCCESS_RESULT), + httpx.Response(200, json=_CR_PARTIAL_SUCCESS_RESULT), + ] + with custom_mp_config(MP_PATH): + with pytest.raises(httpx.HTTPStatusError, match=r"PARTIAL_SUCCESS:.*"): + driver.connect() + driver.query('{"test": "test"}') + + check.is_true(connect.called or driver.connected) + check.is_true(query.called) + check.equal(query.call_count, 3) @respx.mock def test_paginated_query(driver): From 6e453dc4d707f3e94366e1b72397cab95eb094b9 Mon Sep 17 00:00:00 2001 From: "Meroujan.Antonyan" Date: Tue, 22 Oct 2024 08:57:07 +0000 Subject: [PATCH 3/8] doc and keyword params --- msticpy/data/drivers/cybereason_driver.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/msticpy/data/drivers/cybereason_driver.py b/msticpy/data/drivers/cybereason_driver.py index 05edc2f15..a61b92b9c 100644 --- a/msticpy/data/drivers/cybereason_driver.py +++ b/msticpy/data/drivers/cybereason_driver.py @@ -398,6 +398,7 @@ def _create_paginated_query_tasks( def __execute_query( self, body: Dict[str, Any], + *, page: int = 0, page_size: int = 2000, pagination_token: str = None, @@ -406,6 +407,8 @@ def __execute_query( """ Run query with pagination enabled. + :raises httpx.HTTPStatusError: if max_retry reached + Parameters ---------- body: Dict[str, Any] @@ -416,6 +419,8 @@ def __execute_query( Page number to query pagination_token: str Token of the current search + max_retry: int + Maximum retries in case of API no cuccess response Returns ------- From 50e7eee96b7f4cd03efc2e8f34c2bb6054a14f6b Mon Sep 17 00:00:00 2001 From: ianhelle Date: Fri, 22 Nov 2024 11:29:56 +0100 Subject: [PATCH 4/8] Fixing mypy and black warnings --- msticpy/context/azure/azure_data.py | 31 +- msticpy/context/azure/sentinel_utils.py | 2 +- msticpy/context/azure/sentinel_watchlists.py | 6 +- msticpy/context/contextlookup.py | 2 +- msticpy/context/geoip.py | 2 +- msticpy/context/ip_utils.py | 9 +- msticpy/context/provider_base.py | 10 +- msticpy/context/tilookup.py | 2 +- msticpy/context/vtlookupv3/vtfile_behavior.py | 8 +- msticpy/context/vtlookupv3/vtlookup.py | 2 +- msticpy/context/vtlookupv3/vtlookupv3.py | 2 +- msticpy/data/drivers/cybereason_driver.py | 2 +- msticpy/vis/TimelineInteractive.ipynb | 974 ++++++++++++++++++ mypy.ini | 9 + 14 files changed, 1026 insertions(+), 35 deletions(-) create mode 100644 msticpy/vis/TimelineInteractive.ipynb diff --git a/msticpy/context/azure/azure_data.py b/msticpy/context/azure/azure_data.py index 8c048aef9..389b27ca9 100644 --- a/msticpy/context/azure/azure_data.py +++ b/msticpy/context/azure/azure_data.py @@ -818,21 +818,22 @@ def get_network_details( details.network_security_group.id.split("/")[8], ) nsg_rules = [] - for nsg in nsg_details.default_security_rules: - rules = asdict( - NsgItems( - rule_name=nsg.name, - description=nsg.description, - protocol=str(nsg.protocol), - direction=str(nsg.direction), - src_ports=nsg.source_port_range, - dst_ports=nsg.destination_port_range, - src_addrs=nsg.source_address_prefix, - dst_addrs=nsg.destination_address_prefix, - action=str(nsg.access), - ), - ) - nsg_rules.append(rules) + if nsg_details is not None: + for nsg in nsg_details.default_security_rules: + rules = asdict( + NsgItems( + rule_name=nsg.name, + description=nsg.description, + protocol=str(nsg.protocol), + direction=str(nsg.direction), + src_ports=nsg.source_port_range, + dst_ports=nsg.destination_port_range, + src_addrs=nsg.source_address_prefix, + dst_addrs=nsg.destination_address_prefix, + action=str(nsg.access), + ), + ) + nsg_rules.append(rules) nsg_df = pd.DataFrame(nsg_rules) diff --git a/msticpy/context/azure/sentinel_utils.py b/msticpy/context/azure/sentinel_utils.py index cc1c6530f..d9d9095d0 100644 --- a/msticpy/context/azure/sentinel_utils.py +++ b/msticpy/context/azure/sentinel_utils.py @@ -339,7 +339,7 @@ def parse_resource_id(res_id: str) -> dict[str, Any]: """Extract components from workspace resource ID.""" if not res_id.startswith("/"): res_id = f"/{res_id}" - res_id_parts: dict[str, Any] = az_tools.parse_resource_id(res_id) + res_id_parts: dict[str, str] = az_tools.parse_resource_id(res_id) workspace_name: str | None = None if ( res_id_parts.get("namespace") == "Microsoft.OperationalInsights" diff --git a/msticpy/context/azure/sentinel_watchlists.py b/msticpy/context/azure/sentinel_watchlists.py index df8af7e80..a730366e3 100644 --- a/msticpy/context/azure/sentinel_watchlists.py +++ b/msticpy/context/azure/sentinel_watchlists.py @@ -224,12 +224,14 @@ def add_watchlist_item( axis=1, copy=False, ) - if (current_df == item_series).all(axis=1).any() and overwrite: + if (current_df == item_series).all( + axis=1 + ).any() and overwrite: # type: ignore[attr-defined] watchlist_id: str = current_items[ current_items.isin(list(new_item.values())).any(axis=1) ]["properties.watchlistItemId"].iloc[0] # If not in watchlist already generate new ID - elif not (current_df == item_series).all(axis=1).any(): + elif not (current_df == item_series).all(axis=1).any(): # type: ignore[attr-defined] watchlist_id = str(uuid4()) else: err_msg = "Item already exists in the watchlist. Set overwrite = True to replace." diff --git a/msticpy/context/contextlookup.py b/msticpy/context/contextlookup.py index f197fbbfa..631f2f352 100644 --- a/msticpy/context/contextlookup.py +++ b/msticpy/context/contextlookup.py @@ -171,7 +171,7 @@ async def _lookup_observables_async( # pylint:disable=too-many-arguments # noqa ) -> pd.DataFrame: """Lookup items async.""" return await self._lookup_items_async( - data, + data, # type: ignore[arg-type] item_col=obs_col, item_type_col=obs_type_col, query_type=query_type, diff --git a/msticpy/context/geoip.py b/msticpy/context/geoip.py index 1243db783..ed1b0dd87 100644 --- a/msticpy/context/geoip.py +++ b/msticpy/context/geoip.py @@ -588,7 +588,7 @@ def lookup_ip( geo_match = self._get_geomatch_non_public(ip_type) elif self._reader: try: - geo_match = self._reader.city(ip_input).raw + geo_match = self._reader.city(ip_input).raw # type: ignore except (AddressNotFoundError, AttributeError, ValueError): continue if geo_match: diff --git a/msticpy/context/ip_utils.py b/msticpy/context/ip_utils.py index 0f8796a12..3744530eb 100644 --- a/msticpy/context/ip_utils.py +++ b/msticpy/context/ip_utils.py @@ -201,8 +201,11 @@ def create_ip_record( ip_entity.SubscriptionId = ip_hb["SubscriptionId"] geoloc_entity: GeoLocation = GeoLocation() geoloc_entity.CountryOrRegionName = ip_hb["RemoteIPCountry"] - geoloc_entity.Longitude = ip_hb["RemoteIPLongitude"] - geoloc_entity.Latitude = ip_hb["RemoteIPLatitude"] + try: + geoloc_entity.Longitude = float(ip_hb["RemoteIPLongitude"]) + geoloc_entity.Latitude = float(ip_hb["RemoteIPLatitude"]) + except TypeError: + pass ip_entity.Location = geoloc_entity # If Azure network data present add this to host record @@ -493,7 +496,7 @@ def ip_whois( for ip_addr in ip: if rate_limit: sleep(query_rate) - whois_results[ip_addr] = _whois_lookup( + whois_results[ip_addr] = _whois_lookup( # type: ignore[index] ip_addr, raw=raw, retry_count=retry_count, diff --git a/msticpy/context/provider_base.py b/msticpy/context/provider_base.py index b7348849e..1f5bed6c5 100644 --- a/msticpy/context/provider_base.py +++ b/msticpy/context/provider_base.py @@ -20,7 +20,7 @@ from asyncio import get_event_loop from collections.abc import Iterable as C_Iterable from functools import lru_cache, partial, singledispatch -from typing import TYPE_CHECKING, Any, ClassVar, Coroutine, Generator, Iterable +from typing import TYPE_CHECKING, Any, ClassVar, Coroutine, Generator, Iterable, cast import pandas as pd from typing_extensions import Self @@ -368,7 +368,7 @@ def resolve_item_type(item: str) -> str: async def _lookup_items_async_wrapper( # pylint: disable=too-many-arguments # noqa: PLR0913 self: Self, - data: pd.DataFrame | dict[str, str] | list[str], + data: pd.DataFrame | dict[str, str] | Iterable[str], item_col: str | None = None, item_type_col: str | None = None, query_type: str | None = None, @@ -395,7 +395,7 @@ async def _lookup_items_async_wrapper( # pylint: disable=too-many-arguments # n If not specified the default record type for the IoitemC type will be returned. prog_counter: ProgressCounter; Optional - Progress Counter to display progess of IOC searches. + Progress Counter to display progress of IOC searches. Returns ------- @@ -413,7 +413,7 @@ async def _lookup_items_async_wrapper( # pylint: disable=too-many-arguments # n ) result: pd.DataFrame = await event_loop.run_in_executor(None, get_items) if prog_counter: - await prog_counter.decrement(len(data)) + await prog_counter.decrement(len(data)) # type: ignore[arg-type] return result @@ -466,7 +466,7 @@ def generate_items( if isinstance(data, C_Iterable): for item in data: - yield item, Provider.resolve_item_type(item) + yield cast(str, item), Provider.resolve_item_type(item) else: yield None, None diff --git a/msticpy/context/tilookup.py b/msticpy/context/tilookup.py index 7d02fa0be..869975ea0 100644 --- a/msticpy/context/tilookup.py +++ b/msticpy/context/tilookup.py @@ -233,7 +233,7 @@ async def _lookup_iocs_async( # pylint: disable=too-many-arguments #noqa:PLR091 ) -> pd.DataFrame: """Lookup IoCs async.""" return await self._lookup_items_async( - data, + data, # type: ignore[arg-type] item_col=ioc_col, item_type_col=ioc_type_col, query_type=ioc_query_type, diff --git a/msticpy/context/vtlookupv3/vtfile_behavior.py b/msticpy/context/vtlookupv3/vtfile_behavior.py index 41a548af8..b49b1dcb1 100644 --- a/msticpy/context/vtlookupv3/vtfile_behavior.py +++ b/msticpy/context/vtlookupv3/vtfile_behavior.py @@ -146,7 +146,9 @@ def __init__( file_summary = file_summary.iloc[0] if isinstance(file_summary, pd.Series): file_summary = file_summary.to_dict() - self.file_summary: dict[str, Any] = file_summary or {} + self.file_summary: pd.Series | dict[str, pd.Timestamp] | dict[str, Any] = ( + file_summary or {} + ) self.file_id: str | Any | None = file_id or self.file_summary.get("id") self._file_behavior: dict[str, Any] = {} @@ -406,7 +408,7 @@ def _try_match_commandlines( and np.isnan(row["cmd_line"]) and row["name"] in cmd ): - procs_cmd.loc[idx, "cmd_line"] = cmd # type: ignore[reportCallIssue] + procs_cmd.loc[idx, "cmd_line"] = cmd # type: ignore[reportCallIssue, assignment, index] break for cmd in command_executions: for idx, row in procs_cmd.iterrows(): @@ -416,7 +418,7 @@ def _try_match_commandlines( and Path(row["name"]).stem.lower() in cmd.lower() ): weak_matches += 1 - procs_cmd.loc[idx, "cmd_line"] = cmd # type: ignore[reportCallIssue] + procs_cmd.loc[idx, "cmd_line"] = cmd # type: ignore[reportCallIssue, assignment, index] break if weak_matches: diff --git a/msticpy/context/vtlookupv3/vtlookup.py b/msticpy/context/vtlookupv3/vtlookup.py index 4d1dd9848..5762db6e1 100644 --- a/msticpy/context/vtlookupv3/vtlookup.py +++ b/msticpy/context/vtlookupv3/vtlookup.py @@ -823,7 +823,7 @@ def _add_invalid_input_result( new_row["Status"] = status new_row["SourceIndex"] = source_idx new_results: pd.DataFrame = self.results.append( - new_row.to_dict(), + new_row.to_dict(), # type: ignore[operator] ignore_index=True, ) diff --git a/msticpy/context/vtlookupv3/vtlookupv3.py b/msticpy/context/vtlookupv3/vtlookupv3.py index 352cecca1..58c4a168b 100644 --- a/msticpy/context/vtlookupv3/vtlookupv3.py +++ b/msticpy/context/vtlookupv3/vtlookupv3.py @@ -1222,7 +1222,7 @@ def _get_vt_api_key() -> str | None: def timestamps_to_utcdate(data: pd.DataFrame) -> pd.DataFrame: """Replace Unix timestamps in VT data with Py/pandas Timestamp.""" - columns: pd.Index[str] = data.columns + columns: pd.Index = data.columns for date_col in ( col for col in columns if isinstance(col, str) and col.endswith("_date") ): diff --git a/msticpy/data/drivers/cybereason_driver.py b/msticpy/data/drivers/cybereason_driver.py index a61b92b9c..7334e1c94 100644 --- a/msticpy/data/drivers/cybereason_driver.py +++ b/msticpy/data/drivers/cybereason_driver.py @@ -459,7 +459,7 @@ def __execute_query( raise httpx.HTTPStatusError( f"{status}: {json_result['message']}", request=response.request, - response=response + response=response, ) return json_result diff --git a/msticpy/vis/TimelineInteractive.ipynb b/msticpy/vis/TimelineInteractive.ipynb new file mode 100644 index 000000000..63299f17f --- /dev/null +++ b/msticpy/vis/TimelineInteractive.ipynb @@ -0,0 +1,974 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Requirements\n", + "\n", + "1. Add data sources independently\n", + " - [x] Choose from available dataframes (variables)\n", + " - [] prompt for read from file?\n", + " - [ ] Select/edit time range from data\n", + " - [ ] Select time column to use\n", + " - [ ] Select properties for hover\n", + " - [ ] Enter series name\n", + " - [ ] Enter series color\n", + " - [ ] Select order (vertical position)\n", + "\n", + "2. Add annotation:\n", + " - [] Select time\n", + " - [] Choose from event?\n", + " - [] Add title\n", + " - [] Add text\n", + " - [] Tie to data set\n", + " - [] which data set should the annotation reference\n", + " - [] set display offset (to avoid overlaps)\n", + " - [] can we do this dynamically (keep track of what text has gone where)\n", + " - [] combine with \n", + " - [] build into DF\n", + "\n", + "3. Global\n", + " - Save settings\n", + " - Set\n", + " - font size\n", + " - width, height\n", + " - default color\n", + " - theme" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "from functools import singledispatchmethod\n", + "from typing import Any, Dict, Iterable, List, Optional\n", + "\n", + "\n", + "import attr\n", + "from bokeh.models import LayoutDOM\n", + "import pandas as pd\n", + "from msticpy.vis.timeline import display_timeline\n", + "from msticpy.analysis.observationlist import Observations, Observation\n", + "from msticpy.common.timespan import TimeSpan\n", + "\n", + "\n", + "@attr.s(auto_attribs=True)\n", + "class DataSet:\n", + " data: pd.DataFrame\n", + " time_column: str\n", + " series_name: str\n", + " source_columns: attr.Factory(list)\n", + " color: Optional[str] = None\n", + " glyph: str = \"circle\"\n", + "\n", + "\n", + "\n", + "\n", + "class CompositeTimeline:\n", + "\n", + " _TIME_COLUMNS = (\"TimeGenerated\", \"EventTime\", \"timestamp\")\n", + "\n", + " def __init__(\n", + " self,\n", + " data: Optional[pd.DataFrame] = None,\n", + " series_name: Optional[str] = None,\n", + " data_sets: Optional[Dict[str, Any]] = None,\n", + " **kwargs,\n", + " ):\n", + " self._data_sets = Observations()\n", + " self._observations = Observations()\n", + " self._timeline_plot: Optional[LayoutDOM] = None\n", + "\n", + " if data:\n", + " self.add_data(data, series_name, **kwargs)\n", + " if data_sets:\n", + " for ds_series_name, ds_series_attrs in data_sets.items():\n", + " if isinstance(ds_series_attrs, DataSet):\n", + " ds_series_attrs = attr.asdict(ds_series_attrs)\n", + " self.add_data(\n", + " data=ds_series_attrs[\"data\"],\n", + " series_name=ds_series_name,\n", + " time_column=ds_series_attrs[\"time_column\"],\n", + " source_columns=ds_series_attrs.get(\n", + " \"time_column\",\n", + " self._determine_time_column(ds_series_attrs[\"data\"])\n", + " ),\n", + " color=ds_series_attrs.get(\"color\")\n", + " )\n", + "\n", + " @property\n", + " def data_set_names(self) -> List[str]:\n", + " return list(self._data_sets)\n", + " \n", + " def __getitem__(self, key: str) -> Observation:\n", + " return self._data_sets.get(key)\n", + "\n", + " def get_plot_params(self, caption: str) -> DataSet:\n", + " return self._data_sets[caption][\"additional_properties\"][\"plot_params\"]\n", + "\n", + " @singledispatchmethod\n", + " def add_data(\n", + " self,\n", + " data: pd.DataFrame,\n", + " series_name: Optional[str],\n", + " time_column: Optional[str] = None,\n", + " source_columns: Optional[Iterable[str]] = None,\n", + " color: Optional[str] = None,\n", + " glyph: str = \"circle\",\n", + " group_by: Optional[str] = None,\n", + " ) -> None:\n", + " if not time_column:\n", + " time_column = self._determine_time_column(data)\n", + "\n", + " if not time_column:\n", + " raise ValueError(\n", + " \"No value for time_column could be found in the data. \"\n", + " \"Please specify the time column in the 'time_column' parameter.\"\n", + " )\n", + " if not group_by:\n", + " self._add_data_set(\n", + " series_name=series_name,\n", + " data=data,\n", + " time_column=time_column,\n", + " source_columns=source_columns,\n", + " color=color\n", + " )\n", + " else:\n", + " for group, data_group in data.groupby(group_by):\n", + " series_group_name = f\"{series_name} ({group})\"\n", + " self._add_data_set(\n", + " series_name=series_group_name,\n", + " data=data_group,\n", + " time_column=time_column,\n", + " source_columns=source_columns,\n", + " color=color\n", + " )\n", + "\n", + " @add_data.register\n", + " def _(\n", + " self,\n", + " data_set: DataSet,\n", + " ) -> None:\n", + " if not data_set.time_column:\n", + " data_set.time_column = self._determine_time_column(data_set.data)\n", + "\n", + " if not data_set.time_column:\n", + " raise ValueError(\n", + " \"No value for time_column could be found in the data. \"\n", + " \"Please specify the time column in the 'time_column' attribute.\"\n", + " )\n", + " self._datasets[data_set.series_name] = data_set\n", + "\n", + " def _add_data_set(self, series_name, data, time_column, source_columns, color):\n", + " new_data = Observation(\n", + " caption=series_name,\n", + " data=data,\n", + " data_type=\"DataFrame\",\n", + " time_span=TimeSpan(start=data[time_column].min(), end=data[time_column].max()),\n", + " time_column=time_column\n", + " )\n", + " new_data.additional_properties[\"plot_params\"] = DataSet(\n", + " data=data,\n", + " series_name=series_name,\n", + " source_columns=source_columns,\n", + " time_column=time_column,\n", + " color=color,\n", + " )\n", + " self._datasets[series_name] = new_data\n", + "\n", + " def add_annotation(\n", + " self,\n", + " caption: str,\n", + " description: str,\n", + " timestamp: datetime,\n", + " **kwargs,\n", + " ):\n", + " obs_kwargs = {\n", + " key: value for key, value in kwargs.items()\n", + " if key in Observation.all_fields()\n", + " }\n", + " self._observations.add_observation(\n", + " Observation(\n", + " caption=caption, description=description, timestamp=timestamp, **obs_kwargs\n", + " )\n", + " )\n", + "\n", + " def add_observations(self, observations: Iterable[Observation]):\n", + " for observation in observations:\n", + " self.add_observation(observation)\n", + "\n", + " def add_observation(\n", + " self,\n", + " observation: Observation,\n", + " **kwargs,\n", + " ) -> None:\n", + " self._observations.add_observation(observation, **kwargs)\n", + " observation_time = observation.timestamp or kwargs.get(\"timestamp\")\n", + " if not observation_time:\n", + " print(\"No time_stamp supplied, observation will not be plotted.\")\n", + " observation.timestamp = observation_time\n", + "\n", + " def display(self):\n", + "\n", + " # Add normal data series\n", + " timeline_dict = {series_name: attr.asdict(data_set) for series_name, data_set in self._datasets.items()}\n", + "\n", + " # Add observations\n", + " # TODO\n", + "\n", + " def _determine_time_column(self, data: pd.DataFrame) -> Optional[str]:\n", + " return next(iter(\n", + " col for col_lower, col in ((col.casefold(), col)\n", + " for col in data.columns)\n", + " if col_lower in self._TIME_COLUMNS\n", + " ), None)\n", + "\n", + " # def _obs_list_no_expand_to_df(self):\n", + " # \"\"\"Return observations as a DataFrame.\"\"\"\n", + " # obs_list = [\n", + " # attr.asdict(obs, filter=lambda attr, _: attr != \"data\")\n", + " # for obs in self.observation_list.values()\n", + " # if not obs.expand_data\n", + " # ]\n", + " # return pd.json_normalize(obs_list)\n", + "\n", + " # def _obs_list_data_to_df(self):\n", + " # obs_list = [\n", + " # attr.asdict(obs, filter=lambda attr, _: attr != \"data\")\n", + " # for obs in self.observation_list.values()\n", + " # if obs.expand_data\n", + " # ]\n", + " # obs_dfs = [\n", + " # obs.data[[obs.time_column]].rename(columns={obs.time_column, \"time_stamp\"})\n", + " # for obs in self.observation_list.values()\n", + " # if obs.expand_data\n", + " # ]\n", + "\n", + "\n", + "@attr.s(auto_attribs=True):\n", + "\n", + "class TLAnnotation\n", + "\n", + "class TLAnnotationsSet:\n", + "\n", + " def __init__(self):\n", + " self.annotations = List[TLAnnotation]\n", + " def add_annotation()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "dict_keyiterator" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "dt = {1: 2, 3: 4}\n", + "type(iter(dt))" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['__class__',\n", + " '__delattr__',\n", + " '__dict__',\n", + " '__dir__',\n", + " '__doc__',\n", + " '__eq__',\n", + " '__format__',\n", + " '__ge__',\n", + " '__getattribute__',\n", + " '__gt__',\n", + " '__hash__',\n", + " '__init__',\n", + " '__init_subclass__',\n", + " '__le__',\n", + " '__lt__',\n", + " '__module__',\n", + " '__ne__',\n", + " '__new__',\n", + " '__reduce__',\n", + " '__reduce_ex__',\n", + " '__repr__',\n", + " '__setattr__',\n", + " '__sizeof__',\n", + " '__str__',\n", + " '__subclasshook__',\n", + " '__weakref__',\n", + " '_colors']" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from bokeh.colors import groups\n", + "color_groups = groups.__all__\n", + "color_groups[0]\n", + "dir(groups.black)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "49f934ad901b4aa6bf2c4614061e6305", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Button(description='Help', style=ButtonStyle(), tooltip='Confirms updates to the settings changes')" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import ipywidgets as widgets\n", + "\n", + "btn = widgets.Button(description=\"Help\", tooltip=\"Confirms updates to the settings changes\")\n", + "btn" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "39f7d87ec1804454bbc250ff77bc9fc2", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Dropdown(description='Help', options=('one', 'two'), value='one')" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "widgets.Dropdown(description=\"Help\", options=[\"one\", \"two\"], tooltip=\"Confirms updates to the settings changes\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# UI" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "\n", + "import pandas as pd\n", + "\n", + "def get_global_dfs(pattern: str = None):\n", + " \n", + " return [\n", + " name for name in globals()\n", + " if (not pattern or re.match(pattern, name))\n", + " and isinstance(globals()[name], pd.DataFrame)\n", + " and not name.startswith(\"_\")\n", + " ]\n", + "\n", + "var1 = pd.DataFrame()\n", + "xxx = pd.DataFrame()\n", + "\n", + "def get_df_var(var_name: str) -> pd.DataFrame:\n", + " if var_name in globals():\n", + " return globals()[var_name]" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c09e3395dc2d4049a835385565b1d906", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "AppLayout(children=(HTML(value='

Timeline builder

', layout=Layout(grid_area='header')), Label(value='T…" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "import ipywidgets as widgets\n", + "\n", + "\n", + "\n", + "from msticpy.config.file_browser import FileBrowser\n", + "\n", + "\n", + "DESC_WIDTH = {\"style\": {\"description_width\": \"120px\"}}\n", + "\n", + "WIDTH_80 = {\"layout\": widgets.Layout(width=\"80%\")}\n", + "\n", + "def border_layout(width=\"95%\"):\n", + " \"\"\"Return border widget layout.\"\"\"\n", + " return {\n", + " \"layout\": widgets.Layout(\n", + " **{\n", + " \"width\": width,\n", + " \"border\": \"solid gray 1px\",\n", + " \"margin\": \"1pt\",\n", + " \"padding\": \"5pt\",\n", + " }\n", + " )\n", + " }\n", + "\n", + "\n", + "\n", + "\n", + "btn_get_file_name = widgets.Button(description=\"Select file...\", tooltip=\"Browse for DataFrame\")\n", + "FileBrowser()\n", + "\n", + "from msticpy.nbtools.nbwidgets import QueryTime\n", + "\n", + "c_timeline = CompositeTimeline()\n", + "\n", + "\n", + "\n", + "sel_ds = widgets.Select(description=\"Datasets\", options=c_timeline.data_set_names)\n", + "\n", + "# Basic data set edit controls\n", + "txt_ds_caption = widgets.Text(description=\"Series name\", **DESC_WIDTH)\n", + "sel_ds_time_column = widgets.Select(description=\"Time column\", **DESC_WIDTH)\n", + "txt_ds_color = widgets.Text(description=\"Series color\", **DESC_WIDTH)\n", + "sel_ds_groupby = widgets.Select(description=\"Group by\", **DESC_WIDTH)\n", + "msel_ds_tooltips = widgets.SelectMultiple(description=\"Tooltip cols\", **DESC_WIDTH)\n", + "\n", + "\n", + "btn_ds_del = widgets.Button(description=\"Delete\")\n", + "btn_ds_apply = widgets.Button(description=\"Save\")\n", + "vb_ds_edit1 = widgets.VBox([txt_ds_caption, sel_ds_time_column, txt_ds_color])\n", + "vb_ds_edit2 = widgets.VBox([sel_ds_groupby, msel_ds_tooltips])\n", + "hb_ds_edit = widgets.HBox([vb_ds_edit1, vb_ds_edit2, btn_ds_apply], **border_layout(\"70%\"))\n", + "vb_ds_list = widgets.VBox([sel_ds, btn_ds_del], **border_layout(\"25%\"))\n", + "hb_data_set = widgets.HBox([vb_ds_list, hb_ds_edit])\n", + "\n", + "sel_obs = widgets.Select(description=\"Observations\")\n", + "# Observation edit controls\n", + "txt_obs_caption = widgets.Text(description=\"Caption\", **DESC_WIDTH, **WIDTH_80)\n", + "txt_obs_desc = widgets.Textarea(description=\"Text\", **DESC_WIDTH, **WIDTH_80)\n", + "dt_obs_time = widgets.DatePicker(description=\"Timestamp\")\n", + "btn_obs_new = widgets.Button(description=\"New\")\n", + "lbl_obs_sliders = widgets.Label(value=\"Annotation positioning\")\n", + "SLIDER_PARAMS = dict(\n", + " value=0,\n", + " min=-10,\n", + " max=10,\n", + " step=1,\n", + " readout=True,\n", + " continuous_update=False,\n", + " readout_format='d',\n", + " orientation='horizontal',\n", + ")\n", + "int_obs_hoff = widgets.IntSlider(\n", + " description='H Offset:',\n", + " **SLIDER_PARAMS,\n", + ")\n", + "int_obs_voff = widgets.IntSlider(\n", + " description='V Offset:',\n", + " **SLIDER_PARAMS,\n", + ")\n", + "btn_obs_del = widgets.Button(description=\"Delete\")\n", + "btn_obs_save = widgets.Button(description=\"Save\")\n", + "btn_obs_new = widgets.Button(description=\"Save new\")\n", + "btn_obs_clear = widgets.Button(description=\"Clear\")\n", + "hb_obs_buttons = widgets.HBox([btn_obs_save, btn_obs_new, btn_obs_clear])\n", + "vb_obs_edit1 = widgets.VBox([txt_obs_caption, txt_obs_desc, dt_obs_time, hb_obs_buttons], layout=widgets.Layout(width=\"60%\"))\n", + "vb_obs_edit2 = widgets.VBox([lbl_obs_sliders, int_obs_hoff, int_obs_voff])\n", + "hb_obs_edit = widgets.HBox([vb_obs_edit1, vb_obs_edit2], **border_layout(\"65%\"))\n", + "vb_obs_list = widgets.VBox([sel_obs, btn_obs_del], **border_layout(\"25%\"))\n", + "hb_obs = widgets.HBox([vb_obs_list, hb_obs_edit])\n", + "\n", + "\n", + "\n", + "# Add data frame\n", + "sel_add_var_name = widgets.Select(\n", + " description=\"Dataframe variables\",\n", + " options=get_global_dfs(),\n", + " **DESC_WIDTH,\n", + ")\n", + "btn_add_add = widgets.Button(description=\"Add DataFrame\", tooltip=\"Add DataFrame to Datasets.\")\n", + "btn_add_refresh = widgets.Button(description=\"Refresh vars\", tooltip=\"Add DataFrame to Datasets.\")\n", + "hb_add_dataset = widgets.HBox([sel_add_var_name, btn_add_refresh, btn_add_add])\n", + "\n", + "# Filter expression\n", + "txt_df_filter = widgets.Textarea(description=\"Pandas query\", layout=widgets.Layout(height=\"200px\", width=\"50%\"))\n", + "btn_apply_query = widgets.Button(description=\"Apply\", tooltip=\"Apply query to data.\")\n", + "vb_query = widgets.VBox([txt_df_filter, btn_apply_query])\n", + "\n", + "# Time range\n", + "qt_select_time_range = QueryTime(description=\"Filter time range of data\")\n", + "btn_apply_time_range = widgets.Button(description=\"Save\", tooltip=\"Apply selected time range to data.\")\n", + "vb_time_range = widgets.VBox([qt_select_time_range.layout, btn_apply_time_range])\n", + "\n", + "\n", + "accd_filter = widgets.Accordion(children=[vb_time_range, vb_query])\n", + "accd_filter.set_title(0, \"Filter time range\")\n", + "accd_filter.set_title(1, \"Filter data\")\n", + "accd_filter.selected_index = None\n", + "vb_data_set_edit = widgets.VBox([hb_data_set, accd_filter])\n", + "\n", + "tab_ds_obs = widgets.Tab(children=[vb_data_set_edit, hb_obs, hb_add_dataset])\n", + "tab_ds_obs.set_title(0, \"Loaded data sets\")\n", + "tab_ds_obs.set_title(1, \"Annotations\")\n", + "tab_ds_obs.set_title(2, \"Add data sets\")\n", + "\n", + "lbl_status = widgets.Label(value=\"Test status\")\n", + "\n", + "widgets.Layout()\n", + "html_title = widgets.HTML(value=\"

Timeline builder

\", style={\"text-align\": \"center\"})\n", + "app_layout = widgets.AppLayout(\n", + " header=html_title,\n", + " center=tab_ds_obs,\n", + " footer=lbl_status,\n", + " pane_heights=[1, \"400px\", 1]\n", + ")\n", + "\n", + "app_layout" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "## Event handlers\n", + "\n", + "def _refresh_df_vars(btn):\n", + " del btn\n", + " sel_add_var_name.options=get_global_dfs()\n", + "\n", + "btn_add_refresh.on_click(_refresh_df_vars)\n", + "\n", + "def _select_dataset(change):\n", + " df_key = change.get(\"new\")\n", + " data = _get_data_for_key(df_key)\n", + " _update_groupby(data)\n", + " _update_timecolumn(data)\n", + " _update_status(data)\n", + "\n", + "sel_ds.observe(_select_dataset, names=\"value\")\n", + "\n", + "def _save_ds_changes(btn):\n", + " \"\"\"Click handler for save changes to current set.\"\"\"\n", + " if not txt_ds_caption.value:\n", + "\n", + "\n", + "def _update_groupby(data):\n", + " sel_ds_groupby.options = [\n", + " \"None\",\n", + " *(sorted(data.columns))\n", + " ]\n", + "\n", + "def _update_timecolumn(data):\n", + " sel_ds_time_column.options = [\n", + " \"None\",\n", + " *(sorted(data.select_dtypes(include=[\"datetime\", \"datetimetz\"]).columns))\n", + " ]\n", + "\n", + "def _update_status(data):\n", + " # TODO apply filter and timespan\n", + " time_col = sel_ds_time_column.value\n", + " status = (\n", + " f\"Selected records: {len(data)} \"\n", + " f\"Start time: {data[time_col].min().isoformat()} \"\n", + " f\"End time: {data[time_col].max().isoformat()} \"\n", + " )\n", + "\n", + "# stubs\n", + "def _get_data_for_key(key):\n", + " return mde_df\n", + "\n", + "def _update_dataset(caption, time_column, group_by, color, source_columns):\n", + " return True\n" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
TimestampProcessCreationTimeInitiatingProcessCreationTimeInitiatingProcessParentCreationTime
02021-12-15 23:00:03.449464400+00:002021-12-15 23:00:03.449464400+00:002021-11-11 03:26:31.574305+00:002021-11-11 03:26:28.153467700+00:00
12021-12-15 23:00:03.449833900+00:002021-12-15 23:00:03.449833900+00:002021-12-15 23:00:03.038042200+00:002021-11-11 03:26:31.574305+00:00
22021-12-15 23:11:27.173362800+00:002021-12-15 23:11:27.173362800+00:002021-11-11 03:26:32.145838600+00:002021-11-11 03:26:28.153467700+00:00
32021-12-15 23:11:37.869622100+00:002021-12-15 23:11:37.869622100+00:002021-12-15 23:11:26.473328+00:002021-11-11 03:26:32.145838600+00:00
42021-12-15 23:11:37.870024200+00:002021-12-15 23:11:37.870024200+00:002021-12-15 23:11:37.777611600+00:002021-12-15 23:11:26.473328+00:00
...............
1952021-12-16 03:31:11.600695600+00:002021-12-16 03:31:11.600695600+00:002021-12-16 03:28:10.765459700+00:002021-12-16 03:28:07.042663300+00:00
1962021-12-16 03:31:12.486054600+00:002021-12-16 03:31:12.486054600+00:002021-12-16 03:28:07.042663300+00:002021-12-16 03:28:06.996438500+00:00
1972021-12-16 03:32:11.149138600+00:002021-12-16 03:32:11.149138600+00:002021-12-16 03:28:10.930604400+00:002021-12-16 03:28:07.042663300+00:00
1982021-12-16 03:32:14.034770600+00:002021-12-16 03:32:14.034770600+00:002021-12-16 03:28:07.231891200+00:002021-12-16 03:28:07.042663300+00:00
1992021-12-16 03:32:14.040591700+00:002021-12-16 03:32:14.040591700+00:002021-12-16 03:28:07.042663300+00:002021-12-16 03:28:06.996438500+00:00
\n", + "

200 rows × 4 columns

\n", + "
" + ], + "text/plain": [ + " Timestamp ProcessCreationTime \\\n", + "0 2021-12-15 23:00:03.449464400+00:00 2021-12-15 23:00:03.449464400+00:00 \n", + "1 2021-12-15 23:00:03.449833900+00:00 2021-12-15 23:00:03.449833900+00:00 \n", + "2 2021-12-15 23:11:27.173362800+00:00 2021-12-15 23:11:27.173362800+00:00 \n", + "3 2021-12-15 23:11:37.869622100+00:00 2021-12-15 23:11:37.869622100+00:00 \n", + "4 2021-12-15 23:11:37.870024200+00:00 2021-12-15 23:11:37.870024200+00:00 \n", + ".. ... ... \n", + "195 2021-12-16 03:31:11.600695600+00:00 2021-12-16 03:31:11.600695600+00:00 \n", + "196 2021-12-16 03:31:12.486054600+00:00 2021-12-16 03:31:12.486054600+00:00 \n", + "197 2021-12-16 03:32:11.149138600+00:00 2021-12-16 03:32:11.149138600+00:00 \n", + "198 2021-12-16 03:32:14.034770600+00:00 2021-12-16 03:32:14.034770600+00:00 \n", + "199 2021-12-16 03:32:14.040591700+00:00 2021-12-16 03:32:14.040591700+00:00 \n", + "\n", + " InitiatingProcessCreationTime InitiatingProcessParentCreationTime \n", + "0 2021-11-11 03:26:31.574305+00:00 2021-11-11 03:26:28.153467700+00:00 \n", + "1 2021-12-15 23:00:03.038042200+00:00 2021-11-11 03:26:31.574305+00:00 \n", + "2 2021-11-11 03:26:32.145838600+00:00 2021-11-11 03:26:28.153467700+00:00 \n", + "3 2021-12-15 23:11:26.473328+00:00 2021-11-11 03:26:32.145838600+00:00 \n", + "4 2021-12-15 23:11:37.777611600+00:00 2021-12-15 23:11:26.473328+00:00 \n", + ".. ... ... \n", + "195 2021-12-16 03:28:10.765459700+00:00 2021-12-16 03:28:07.042663300+00:00 \n", + "196 2021-12-16 03:28:07.042663300+00:00 2021-12-16 03:28:06.996438500+00:00 \n", + "197 2021-12-16 03:28:10.930604400+00:00 2021-12-16 03:28:07.042663300+00:00 \n", + "198 2021-12-16 03:28:07.231891200+00:00 2021-12-16 03:28:07.042663300+00:00 \n", + "199 2021-12-16 03:28:07.042663300+00:00 2021-12-16 03:28:06.996438500+00:00 \n", + "\n", + "[200 rows x 4 columns]" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mde_df.select_dtypes(include=[\"datetime\", \"datetimetz\"])\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def get_matching_vars\n", + "combo_df_var_name.observe(get_matching_vars, names=\"value\")" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " Loading BokehJS ...\n", + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n const JS_MIME_TYPE = 'application/javascript';\n const HTML_MIME_TYPE = 'text/html';\n const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n const CLASS_NAME = 'output_bokeh rendered_html';\n\n /**\n * Render data to the DOM node\n */\n function render(props, node) {\n const script = document.createElement(\"script\");\n node.appendChild(script);\n }\n\n /**\n * Handle when an output is cleared or removed\n */\n function handleClearOutput(event, handle) {\n const cell = handle.cell;\n\n const id = cell.output_area._bokeh_element_id;\n const server_id = cell.output_area._bokeh_server_id;\n // Clean up Bokeh references\n if (id != null && id in Bokeh.index) {\n Bokeh.index[id].model.document.clear();\n delete Bokeh.index[id];\n }\n\n if (server_id !== undefined) {\n // Clean up Bokeh references\n const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n cell.notebook.kernel.execute(cmd_clean, {\n iopub: {\n output: function(msg) {\n const id = msg.content.text.trim();\n if (id in Bokeh.index) {\n Bokeh.index[id].model.document.clear();\n delete Bokeh.index[id];\n }\n }\n }\n });\n // Destroy server and session\n const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n cell.notebook.kernel.execute(cmd_destroy);\n }\n }\n\n /**\n * Handle when a new output is added\n */\n function handleAddOutput(event, handle) {\n const output_area = handle.output_area;\n const output = handle.output;\n\n // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n return\n }\n\n const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n\n if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n // store reference to embed id on output_area\n output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n }\n if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n const bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n const script_attrs = bk_div.children[0].attributes;\n for (let i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n }\n\n function register_renderer(events, OutputArea) {\n\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n const toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[toinsert.length - 1]);\n element.append(toinsert);\n return toinsert\n }\n\n /* Handle when an output is cleared or removed */\n events.on('clear_output.CodeCell', handleClearOutput);\n events.on('delete.Cell', handleClearOutput);\n\n /* Handle when a new output is added */\n events.on('output_added.OutputArea', handleAddOutput);\n\n /**\n * Register the mime type and append_mime function with output_area\n */\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n /* Is output safe? */\n safe: true,\n /* Index of renderer in `output_area.display_order` */\n index: 0\n });\n }\n\n // register the mime type if in Jupyter Notebook environment and previously unregistered\n if (root.Jupyter !== undefined) {\n const events = require('base/js/events');\n const OutputArea = require('notebook/js/outputarea').OutputArea;\n\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n }\n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"
    \\n\"+\n \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n \"
  • use INLINE resources instead, as so:
    • \\n\"+\n \"
\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n const el = document.getElementById(\"1002\");\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n \n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.2.min.js\"];\n const css_urls = [];\n \n\n const inline_js = [\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n function(Bokeh) {\n \n \n }\n ];\n\n function run_inline_js() {\n \n if (root.Bokeh !== undefined || force === true) {\n \n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n if (force === true) {\n display_loaded();\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(\"1002\")).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));", + "application/vnd.bokehjs_load.v0+json": "" + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/javascript": "(function(root) {\n function embed_document(root) {\n \n const docs_json = {\"d6e21d0a-d676-473e-948e-daea2d8fbf3f\":{\"defs\":[],\"roots\":{\"references\":[{\"attributes\":{\"children\":[{\"id\":\"1005\"},{\"id\":\"1037\"}]},\"id\":\"1075\",\"type\":\"Column\"},{\"attributes\":{\"num_minor_ticks\":5,\"tickers\":[{\"id\":\"1101\"},{\"id\":\"1102\"},{\"id\":\"1103\"},{\"id\":\"1104\"},{\"id\":\"1105\"},{\"id\":\"1106\"},{\"id\":\"1107\"},{\"id\":\"1108\"},{\"id\":\"1109\"},{\"id\":\"1110\"},{\"id\":\"1111\"},{\"id\":\"1112\"}]},\"id\":\"1049\",\"type\":\"DatetimeTicker\"},{\"attributes\":{\"active_multi\":{\"id\":\"1062\"},\"tools\":[{\"id\":\"1062\"}]},\"id\":\"1052\",\"type\":\"Toolbar\"},{\"attributes\":{\"align\":\"right\",\"coordinates\":null,\"group\":null,\"text\":\"Drag the middle or edges of the selection box to change the range in the main chart\",\"text_font_size\":\"10px\"},\"id\":\"1053\",\"type\":\"Title\"},{\"attributes\":{\"months\":[0,4,8]},\"id\":\"1097\",\"type\":\"MonthsTicker\"},{\"attributes\":{\"axis\":{\"id\":\"1048\"},\"coordinates\":null,\"group\":null,\"ticker\":null},\"id\":\"1051\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1085\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1086\",\"type\":\"Selection\"},{\"attributes\":{\"coordinates\":null,\"group\":null,\"text\":\"Range Selector\"},\"id\":\"1038\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1082\",\"type\":\"AllLabels\"},{\"attributes\":{\"days\":[1,8,15,22]},\"id\":\"1093\",\"type\":\"DaysTicker\"},{\"attributes\":{},\"id\":\"1080\",\"type\":\"AllLabels\"},{\"attributes\":{},\"id\":\"1044\",\"type\":\"LinearScale\"},{\"attributes\":{\"months\":[0,6]},\"id\":\"1098\",\"type\":\"MonthsTicker\"},{\"attributes\":{},\"id\":\"1046\",\"type\":\"LinearScale\"},{\"attributes\":{\"coordinates\":null,\"formatter\":{\"id\":\"1054\"},\"group\":null,\"major_label_policy\":{\"id\":\"1084\"},\"ticker\":{\"id\":\"1049\"}},\"id\":\"1048\",\"type\":\"DatetimeAxis\"},{\"attributes\":{\"months\":[0,1,2,3,4,5,6,7,8,9,10,11]},\"id\":\"1095\",\"type\":\"MonthsTicker\"},{\"attributes\":{},\"id\":\"1042\",\"type\":\"DataRange1d\"},{\"attributes\":{\"days\":[1,15]},\"id\":\"1094\",\"type\":\"DaysTicker\"},{\"attributes\":{\"months\":[0,2,4,6,8,10]},\"id\":\"1096\",\"type\":\"MonthsTicker\"},{\"attributes\":{\"days\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]},\"id\":\"1091\",\"type\":\"DaysTicker\"},{\"attributes\":{},\"id\":\"1079\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"months\":[0,6]},\"id\":\"1111\",\"type\":\"MonthsTicker\"},{\"attributes\":{\"days\":[1,4,7,10,13,16,19,22,25,28]},\"id\":\"1092\",\"type\":\"DaysTicker\"},{\"attributes\":{\"fill_color\":{\"value\":\"navy\"},\"hatch_color\":{\"value\":\"navy\"},\"line_color\":{\"value\":\"navy\"},\"x\":{\"field\":\"Timestamp\"},\"y\":{\"field\":\"y_index\"}},\"id\":\"1057\",\"type\":\"Circle\"},{\"attributes\":{},\"id\":\"1112\",\"type\":\"YearsTicker\"},{\"attributes\":{\"months\":[0,4,8]},\"id\":\"1110\",\"type\":\"MonthsTicker\"},{\"attributes\":{},\"id\":\"1099\",\"type\":\"YearsTicker\"},{\"attributes\":{\"source\":{\"id\":\"1003\"}},\"id\":\"1074\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1026\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1021\",\"type\":\"BasicTicker\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1024\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"coordinates\":null,\"formatter\":{\"id\":\"1079\"},\"group\":null,\"major_label_policy\":{\"id\":\"1080\"},\"ticker\":{\"id\":\"1021\"},\"visible\":false},\"id\":\"1020\",\"type\":\"LinearAxis\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"navy\"},\"hatch_alpha\":{\"value\":0.1},\"hatch_color\":{\"value\":\"navy\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"navy\"},\"marker\":{\"value\":\"diamond\"},\"size\":{\"value\":10},\"x\":{\"field\":\"Timestamp\"},\"y\":{\"field\":\"y_index\"}},\"id\":\"1071\",\"type\":\"Scatter\"},{\"attributes\":{\"data\":{\"Timestamp\":{\"__ndarray__\":\"bZevLwXcd0JUna8vBdx3QstVnNYF3HdC9Nk42QXcd0Ji4DjZBdx3QuleG9sF3HdCpoEb2wXcd0LVmE1hB9x3QlBdZGMH3HdCFJSeYwfcd0KH9ASeCNx3QrhGaxAJ3HdCO01rEAncd0KHjv5JCdx3Qqp5RTYK3HdCUh5A0Arcd0LFRiPKC9x3QvyLrMwL3HdCfZGszAvcd0KR5ejOC9x3QhuT9M4L3HdCI1kd0Qvcd0II3PN5DNx3QnHj83kM3HdCWMGQewzcd0LbX757DNx3QlilvnsM3HdCYlAIPw7cd0KHhgg/Dtx3QhfNN8kO3HdCh9I3yQ7cd0KWQ1P2D9x3QhmS6a0R3HdCpFL1TxLcd0KwFpxSEtx3QvwbnFIS3HdCTsaNVBLcd0IX+41UEtx3Qt/fgeoS3HdC152i6xLcd0LDVe7rEtx3QmJm7usS3HdCkVvK7hLcd0ItwlXvEtx3QpzsLvIS3HdCrLzp+xLcd0I/acn8Etx3Qt0mUAUT3HdCouvBBRPcd0IExhQME9x3QsMpygwT3HdC2y/6HhPcd0JvstFhE9x3Qjk81GET3HdCuHB0gxPcd0JUo5+fE9x3Qo2pn58T3HdCVK2fnxPcd0KBs5+fE9x3Qvy3n58T3HdCnr2fnxPcd0Izz5+fE9x3QiXUn58T3HdCuvF4oxPcd0KHptnGE9x3Qjfp48YT3HdCEuOY9hPcd0JvDmL3E9x3Ql4MpY8U3HdC03uljxTcd0L+qKWPFNx3Qjsjpo8U3HdCrCamjxTcd0KLRqaPFNx3Qgxepo8U3HdCNairjxTcd0KJpa6PFNx3QnXBv48U3HdCZiLAjxTcd0LwT8CPFNx3Qn3twI8U3HdCBsHVjxTcd0KuzdaPFNx3Qucp148U3HdCP2XfjxTcd0LufN+PFNx3QvaM348U3HdCFDzijxTcd0KuA+OPFNx3QjF2448U3HdCmkfkjxTcd0KRVeSPFNx3Qvxh5I8U3HdCf2jkjxTcd0LwbeSPFNx3Qp6B5I8U3HdCpofkjxTcd0Lsi+SPFNx3QhBE5Y8U3HdC7FnljxTcd0L4XeWPFNx3QmJi5Y8U3HdCnGbljxTcd0KsauWPFNx3Qi9v5Y8U3HdCbXXljxTcd0LfeeWPFNx3QhB+5Y8U3HdCyYLljxTcd0I7h+WPFNx3QiuN5Y8U3HdClpHljxTcd0JzluWPFNx3Qkib5Y8U3HdCmp/ljxTcd0J1o+WPFNx3Qomn5Y8U3HdClqvljxTcd0ICseWPFNx3QkjB5Y8U3HdCWMfljxTcd0I/zeWPFNx3Qr7R5Y8U3HdCUNvljxTcd0LP3+WPFNx3QhLl5Y8U3HdC8urljxTcd0Lu7uWPFNx3Qkzz5Y8U3HdCXP/ljxTcd0JaGuaPFNx3Qk4e5o8U3HdCGSLmjxTcd0JOJuaPFNx3QhQq5o8U3HdCqi3mjxTcd0L0MeaPFNx3Qpo15o8U3HdCujnmjxTcd0J7cOaPFNx3Qht15o8U3HdC2X7mjxTcd0IKg+aPFNx3Qh+H5o8U3HdCUozmjxTcd0K4nOaPFNx3QrCg5o8U3HdCoLTmjxTcd0J1veaPFNx3QnPE5o8U3HdCqMrmjxTcd0KNz+aPFNx3QtfT5o8U3HdCPdjmjxTcd0Ko3OaPFNx3Qrbt5o8U3HdC5/HmjxTcd0IA9uaPFNx3Qgz65o8U3HdCTv7mjxTcd0JSAuePFNx3QikQ548U3HdCbRnnjxTcd0K8JuePFNx3Qkwr548U3HdCMTbnjxTcd0IERuePFNx3QotK548U3HdCvm3njxTcd0KJceePFNx3Qlh9548U3HdCiYHnjxTcd0J9h+ePFNx3QsWg548U3HdCVLfnjxTcd0IvL+mPFNx3QuwL7o8U3HdCNfrujxTcd0IrXRSQFNx3QgxmGpAU3HdCUMGXkBTcd0KHBtGQFNx3Qh8n0ZAU3HdCz/VBkRTcd0IxWNqkFNx3QkTr2qQU3HdCQrz9pBTcd0LsA/6kFNx3QgyUEKUU3HdCVkwzpRTcd0IhSJalFNx3QmB7lqUU3HdCh/JIsxTcd0J9AUmzFNx3Qs0oSbMU3HdCHwtnsxTcd0LdYJ6zFNx3QjXS8MEU3HdCUiylwhTcd0J1iaXCFNx3Qg==\",\"dtype\":\"float64\",\"order\":\"little\",\"shape\":[200]},\"index\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199],\"y_index\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},\"selected\":{\"id\":\"1086\"},\"selection_policy\":{\"id\":\"1085\"}},\"id\":\"1003\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"mantissas\":[1,2,5],\"max_interval\":500.0,\"num_minor_ticks\":0},\"id\":\"1101\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"navy\"},\"hatch_alpha\":{\"value\":0.5},\"hatch_color\":{\"value\":\"navy\"},\"line_alpha\":{\"value\":0.5},\"line_color\":{\"value\":\"navy\"},\"marker\":{\"value\":\"diamond\"},\"size\":{\"value\":10},\"x\":{\"field\":\"Timestamp\"},\"y\":{\"field\":\"y_index\"}},\"id\":\"1070\",\"type\":\"Scatter\"},{\"attributes\":{\"months\":[0,1,2,3,4,5,6,7,8,9,10,11]},\"id\":\"1108\",\"type\":\"MonthsTicker\"},{\"attributes\":{\"start\":-1.0},\"id\":\"1010\",\"type\":\"Range1d\"},{\"attributes\":{\"coordinates\":null,\"data_source\":{\"id\":\"1003\"},\"glyph\":{\"id\":\"1070\"},\"group\":null,\"hover_glyph\":null,\"muted_glyph\":{\"id\":\"1072\"},\"nonselection_glyph\":{\"id\":\"1071\"},\"view\":{\"id\":\"1074\"}},\"id\":\"1073\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"days\":[1,4,7,10,13,16,19,22,25,28]},\"id\":\"1105\",\"type\":\"DaysTicker\"},{\"attributes\":{},\"id\":\"1084\",\"type\":\"AllLabels\"},{\"attributes\":{\"num_minor_ticks\":10,\"tickers\":[{\"id\":\"1088\"},{\"id\":\"1089\"},{\"id\":\"1090\"},{\"id\":\"1091\"},{\"id\":\"1092\"},{\"id\":\"1093\"},{\"id\":\"1094\"},{\"id\":\"1095\"},{\"id\":\"1096\"},{\"id\":\"1097\"},{\"id\":\"1098\"},{\"id\":\"1099\"}]},\"id\":\"1017\",\"type\":\"DatetimeTicker\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"navy\"},\"hatch_alpha\":{\"value\":0.1},\"hatch_color\":{\"value\":\"navy\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"navy\"},\"x\":{\"field\":\"Timestamp\"},\"y\":{\"field\":\"y_index\"}},\"id\":\"1058\",\"type\":\"Circle\"},{\"attributes\":{\"days\":[1,15]},\"id\":\"1107\",\"type\":\"DaysTicker\"},{\"attributes\":{\"coordinates\":null,\"data_source\":{\"id\":\"1003\"},\"glyph\":{\"id\":\"1057\"},\"group\":null,\"hover_glyph\":null,\"muted_glyph\":{\"id\":\"1059\"},\"nonselection_glyph\":{\"id\":\"1058\"},\"view\":{\"id\":\"1061\"}},\"id\":\"1060\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"source\":{\"id\":\"1003\"}},\"id\":\"1061\",\"type\":\"CDSView\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.2},\"fill_color\":{\"value\":\"navy\"},\"hatch_alpha\":{\"value\":0.2},\"hatch_color\":{\"value\":\"navy\"},\"line_alpha\":{\"value\":0.2},\"line_color\":{\"value\":\"navy\"},\"x\":{\"field\":\"Timestamp\"},\"y\":{\"field\":\"y_index\"}},\"id\":\"1059\",\"type\":\"Circle\"},{\"attributes\":{},\"id\":\"1027\",\"type\":\"SaveTool\"},{\"attributes\":{\"end\":1639627983629.2607,\"start\":1639606753860.7954},\"id\":\"1040\",\"type\":\"Range1d\"},{\"attributes\":{\"days\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]},\"id\":\"1104\",\"type\":\"DaysTicker\"},{\"attributes\":{\"axis\":{\"id\":\"1020\"},\"coordinates\":null,\"dimension\":1,\"grid_line_color\":null,\"group\":null,\"ticker\":null},\"id\":\"1023\",\"type\":\"Grid\"},{\"attributes\":{\"days\":[\"%m-%d %H:%M\"],\"hours\":[\"%H:%M:%S\"],\"milliseconds\":[\"%H:%M:%S.%3N\"],\"minutes\":[\"%H:%M:%S\"],\"seconds\":[\"%H:%M:%S\"]},\"id\":\"1067\",\"type\":\"DatetimeTickFormatter\"},{\"attributes\":{\"below\":[{\"id\":\"1016\"}],\"center\":[{\"id\":\"1019\"},{\"id\":\"1023\"}],\"height\":300,\"left\":[{\"id\":\"1020\"}],\"min_border_left\":50,\"renderers\":[{\"id\":\"1073\"}],\"title\":{\"id\":\"1006\"},\"toolbar\":{\"id\":\"1030\"},\"width\":900,\"x_range\":{\"id\":\"1008\"},\"x_scale\":{\"id\":\"1012\"},\"y_range\":{\"id\":\"1010\"},\"y_scale\":{\"id\":\"1014\"}},\"id\":\"1005\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"base\":24,\"mantissas\":[1,2,4,6,8,12],\"max_interval\":43200000.0,\"min_interval\":3600000.0,\"num_minor_ticks\":0},\"id\":\"1103\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"days\":[\"%m-%d %H:%M\"],\"hours\":[\"%H:%M:%S\"],\"milliseconds\":[\"%H:%M:%S.%3N\"],\"minutes\":[\"%H:%M:%S\"],\"seconds\":[\"%H:%M:%S\"]},\"id\":\"1054\",\"type\":\"DatetimeTickFormatter\"},{\"attributes\":{\"base\":60,\"mantissas\":[1,2,5,10,15,20,30],\"max_interval\":1800000.0,\"min_interval\":1000.0,\"num_minor_ticks\":0},\"id\":\"1102\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"end\":1639627167099.7043,\"start\":1639607570390.3516},\"id\":\"1008\",\"type\":\"Range1d\"},{\"attributes\":{\"axis\":{\"id\":\"1016\"},\"coordinates\":null,\"group\":null,\"minor_grid_line_alpha\":0.3,\"minor_grid_line_color\":\"navy\",\"ticker\":null},\"id\":\"1019\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1014\",\"type\":\"LinearScale\"},{\"attributes\":{\"overlay\":{\"id\":\"1063\"},\"x_range\":{\"id\":\"1008\"},\"y_range\":null},\"id\":\"1062\",\"type\":\"RangeTool\"},{\"attributes\":{\"days\":[1,8,15,22]},\"id\":\"1106\",\"type\":\"DaysTicker\"},{\"attributes\":{\"bottom_units\":\"screen\",\"coordinates\":null,\"fill_alpha\":0.5,\"fill_color\":\"lightgrey\",\"group\":null,\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":1.0,\"line_color\":\"black\",\"line_dash\":[4,4],\"line_width\":2,\"right_units\":\"screen\",\"syncable\":false,\"top_units\":\"screen\"},\"id\":\"1029\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"base\":24,\"mantissas\":[1,2,4,6,8,12],\"max_interval\":43200000.0,\"min_interval\":3600000.0,\"num_minor_ticks\":0},\"id\":\"1090\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.2},\"fill_color\":{\"value\":\"navy\"},\"hatch_alpha\":{\"value\":0.2},\"hatch_color\":{\"value\":\"navy\"},\"line_alpha\":{\"value\":0.2},\"line_color\":{\"value\":\"navy\"},\"marker\":{\"value\":\"diamond\"},\"size\":{\"value\":10},\"x\":{\"field\":\"Timestamp\"},\"y\":{\"field\":\"y_index\"}},\"id\":\"1072\",\"type\":\"Scatter\"},{\"attributes\":{\"coordinates\":null,\"fill_alpha\":0.2,\"fill_color\":\"navy\",\"group\":null,\"level\":\"overlay\",\"line_alpha\":1.0,\"line_color\":\"black\",\"line_dash\":[2,2],\"line_width\":0.5,\"syncable\":false},\"id\":\"1063\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"months\":[0,2,4,6,8,10]},\"id\":\"1109\",\"type\":\"MonthsTicker\"},{\"attributes\":{\"base\":60,\"mantissas\":[1,2,5,10,15,20,30],\"max_interval\":1800000.0,\"min_interval\":1000.0,\"num_minor_ticks\":0},\"id\":\"1089\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"mantissas\":[1,2,5],\"max_interval\":500.0,\"num_minor_ticks\":0},\"id\":\"1088\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"overlay\":{\"id\":\"1029\"}},\"id\":\"1025\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"axis_label\":\"Event Time\",\"coordinates\":null,\"formatter\":{\"id\":\"1067\"},\"group\":null,\"major_label_policy\":{\"id\":\"1082\"},\"ticker\":{\"id\":\"1017\"}},\"id\":\"1016\",\"type\":\"DatetimeAxis\"},{\"attributes\":{\"tools\":[{\"id\":\"1004\"},{\"id\":\"1024\"},{\"id\":\"1025\"},{\"id\":\"1026\"},{\"id\":\"1027\"},{\"id\":\"1028\"}]},\"id\":\"1030\",\"type\":\"Toolbar\"},{\"attributes\":{\"below\":[{\"id\":\"1048\"},{\"id\":\"1053\"}],\"center\":[{\"id\":\"1051\"}],\"height\":120,\"renderers\":[{\"id\":\"1060\"}],\"title\":{\"id\":\"1038\"},\"toolbar\":{\"id\":\"1052\"},\"toolbar_location\":null,\"width\":900,\"x_range\":{\"id\":\"1040\"},\"x_scale\":{\"id\":\"1044\"},\"y_range\":{\"id\":\"1042\"},\"y_scale\":{\"id\":\"1046\"}},\"id\":\"1037\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1028\",\"type\":\"PanTool\"},{\"attributes\":{\"coordinates\":null,\"group\":null,\"text\":\"Event Timeline\"},\"id\":\"1006\",\"type\":\"Title\"},{\"attributes\":{\"callback\":null,\"formatters\":{\"@Timestamp\":\"datetime\"},\"tooltips\":[[\"Timestamp\",\"@Timestamp{%F %T.%3N}\"]]},\"id\":\"1004\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"1012\",\"type\":\"LinearScale\"}],\"root_ids\":[\"1075\"]},\"title\":\"Bokeh Application\",\"version\":\"2.4.2\"}};\n const render_items = [{\"docid\":\"d6e21d0a-d676-473e-948e-daea2d8fbf3f\",\"root_ids\":[\"1075\"],\"roots\":{\"1075\":\"359feaf4-3ee5-4c0b-9a03-5a15d59d26f0\"}}];\n root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n\n }\n if (root.Bokeh !== undefined) {\n embed_document(root);\n } else {\n let attempts = 0;\n const timer = setInterval(function(root) {\n if (root.Bokeh !== undefined) {\n clearInterval(timer);\n embed_document(root);\n } else {\n attempts++;\n if (attempts > 100) {\n clearInterval(timer);\n console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n }\n }\n }, 10, root)\n }\n})(window);", + "application/vnd.bokehjs_exec.v0+json": "" + }, + "metadata": { + "application/vnd.bokehjs_exec.v0+json": { + "id": "1075" + } + }, + "output_type": "display_data" + } + ], + "source": [ + "from msticpy.vis.timeline import display_timeline\n", + "\n", + "mde_df = pd.read_pickle(\"e:/src/msticpy/tests/testdata/mde_proc_pub.pkl\")\n", + "\n", + "plot = display_timeline(data=mde_df, time_column=\"Timestamp\")" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Figure(
id = '1005', …)
above = [],
align = 'start',
aspect_ratio = None,
aspect_scale = 1,
background = None,
background_fill_alpha = 1.0,
background_fill_color = '#ffffff',
below = [DatetimeAxis(id='1016', ...)],
border_fill_alpha = 1.0,
border_fill_color = '#ffffff',
center = [Grid(id='1019', ...), Grid(id='1023', ...)],
css_classes = [],
disabled = False,
extra_x_ranges = {},
extra_x_scales = {},
extra_y_ranges = {},
extra_y_scales = {},
frame_height = None,
frame_width = None,
height = 300,
height_policy = 'auto',
hidpi = True,
inner_height = 0,
inner_width = 0,
js_event_callbacks = {},
js_property_callbacks = {},
left = [LinearAxis(id='1020', ...)],
lod_factor = 10,
lod_interval = 300,
lod_threshold = 2000,
lod_timeout = 500,
margin = (0, 0, 0, 0),
match_aspect = False,
max_height = None,
max_width = None,
min_border = 5,
min_border_bottom = None,
min_border_left = 50,
min_border_right = None,
min_border_top = None,
min_height = None,
min_width = None,
name = None,
outer_height = 0,
outer_width = 0,
outline_line_alpha = 1.0,
outline_line_cap = 'butt',
outline_line_color = '#e5e5e5',
outline_line_dash = [],
outline_line_dash_offset = 0,
outline_line_join = 'bevel',
outline_line_width = 1,
output_backend = 'canvas',
renderers = [GlyphRenderer(id='1073', ...)],
reset_policy = 'standard',
right = [],
sizing_mode = None,
subscribed_events = [],
syncable = True,
tags = [],
title = Title(id='1006', ...),
title_location = 'above',
toolbar = Toolbar(id='1030', ...),
toolbar_location = 'right',
toolbar_sticky = True,
visible = True,
width = 900,
width_policy = 'auto',
x_range = Range1d(id='1008', ...),
x_scale = LinearScale(id='1012', ...),
y_range = Range1d(id='1010', ...),
y_scale = LinearScale(id='1014', ...))
\n", + "\n" + ], + "text/plain": [ + "Figure(id='1005', ...)" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "[attr for attr in dir(plot) if not attr.startswith(\"_\")]\n", + "fig = plot.children[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "7bdcf5ad89444b7c89e9883e24ce175c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "IntSlider(value=0, continuous_update=False, description='V Offset:', max=10, min=-10, orientation='vertical')" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "widgets.IntSlider(\n", + " value=0,\n", + " min=-10,\n", + " max=10,\n", + " step=1,\n", + " description='H Offset:',\n", + " disabled=False,\n", + " continuous_update=False,\n", + " orientation='horizontal',\n", + " readout=True,\n", + " readout_format='d'\n", + ")\n", + "widgets.IntSlider(\n", + " value=0,\n", + " min=-10,\n", + " max=10,\n", + " step=1,\n", + " description='V Offset:',\n", + " disabled=False,\n", + " continuous_update=False,\n", + " orientation='vertical',\n", + " readout=True,\n", + " readout_format='d'\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "msticpy", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13 (default, Mar 28 2022, 06:59:08) [MSC v.1916 64 bit (AMD64)]" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "4b6ddaa4f4daa4a6c6674a452b84f1f4f3e0236ad39c89274d27c46b8457b97f" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/mypy.ini b/mypy.ini index e31cc92db..021d7e8a0 100644 --- a/mypy.ini +++ b/mypy.ini @@ -143,3 +143,12 @@ ignore_missing_imports = True [mypy-requests.*] ignore_missing_imports = True + +[mypy-autogen.*] +ignore_missing_imports = True + +[mypy-notebookutils.*] +ignore_missing_imports = True + +[mypy-mo_sql_parsing.*] +ignore_missing_imports = True From 3aefe3ccd119d0ff66c649ce544d8f242b9fd062 Mon Sep 17 00:00:00 2001 From: ianhelle Date: Fri, 22 Nov 2024 11:50:36 +0100 Subject: [PATCH 5/8] Updating pre-commit yaml --- .pre-commit-config.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a794c8889..e957ab5c3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v5.0.0 hooks: - id: check-yaml - id: check-json @@ -8,19 +8,19 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - repo: https://github.com/ambv/black - rev: 24.4.2 + rev: 24.10.0 hooks: - id: black language: python - repo: https://github.com/PyCQA/pylint - rev: v3.2.2 + rev: v3.3.1 hooks: - id: pylint args: - --disable=duplicate-code,import-error - --ignore-patterns=test_ - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 7.1.1 hooks: - id: flake8 args: @@ -28,7 +28,7 @@ repos: - --max-line-length=90 - --exclude=tests,test*.py - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort name: isort (python) @@ -43,7 +43,7 @@ repos: - --convention=numpy - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.7.0 + rev: v0.8.0 hooks: # Run the linter. - id: ruff From 6ca58b8029a688a1b82a5e91ac81fa0392018add Mon Sep 17 00:00:00 2001 From: ianhelle Date: Fri, 22 Nov 2024 12:31:43 +0100 Subject: [PATCH 6/8] Fix pylint and mypy warnings --- msticpy/context/azure/azure_data.py | 2 +- msticpy/context/azure/sentinel_utils.py | 6 ++++-- msticpy/context/vtlookupv3/vtfile_behavior.py | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/msticpy/context/azure/azure_data.py b/msticpy/context/azure/azure_data.py index 389b27ca9..2eb3cbd68 100644 --- a/msticpy/context/azure/azure_data.py +++ b/msticpy/context/azure/azure_data.py @@ -819,7 +819,7 @@ def get_network_details( ) nsg_rules = [] if nsg_details is not None: - for nsg in nsg_details.default_security_rules: + for nsg in nsg_details.default_security_rules: # type: ignore rules = asdict( NsgItems( rule_name=nsg.name, diff --git a/msticpy/context/azure/sentinel_utils.py b/msticpy/context/azure/sentinel_utils.py index d9d9095d0..4ffc018ec 100644 --- a/msticpy/context/azure/sentinel_utils.py +++ b/msticpy/context/azure/sentinel_utils.py @@ -15,7 +15,7 @@ import pandas as pd from azure.common.exceptions import CloudError from azure.mgmt.core import tools as az_tools -from typing_extensions import Self +from typing_extensions import Self, cast from ..._version import VERSION from ...auth.azure_auth_core import AzureCloudConfig @@ -339,7 +339,9 @@ def parse_resource_id(res_id: str) -> dict[str, Any]: """Extract components from workspace resource ID.""" if not res_id.startswith("/"): res_id = f"/{res_id}" - res_id_parts: dict[str, str] = az_tools.parse_resource_id(res_id) + res_id_parts: dict[str, str] = cast( + dict[str, str], az_tools.parse_resource_id(res_id) + ) workspace_name: str | None = None if ( res_id_parts.get("namespace") == "Microsoft.OperationalInsights" diff --git a/msticpy/context/vtlookupv3/vtfile_behavior.py b/msticpy/context/vtlookupv3/vtfile_behavior.py index b49b1dcb1..4d9694839 100644 --- a/msticpy/context/vtlookupv3/vtfile_behavior.py +++ b/msticpy/context/vtlookupv3/vtfile_behavior.py @@ -408,7 +408,7 @@ def _try_match_commandlines( and np.isnan(row["cmd_line"]) and row["name"] in cmd ): - procs_cmd.loc[idx, "cmd_line"] = cmd # type: ignore[reportCallIssue, assignment, index] + procs_cmd.loc[idx, "cmd_line"] = cmd # type: ignore break for cmd in command_executions: for idx, row in procs_cmd.iterrows(): @@ -418,7 +418,7 @@ def _try_match_commandlines( and Path(row["name"]).stem.lower() in cmd.lower() ): weak_matches += 1 - procs_cmd.loc[idx, "cmd_line"] = cmd # type: ignore[reportCallIssue, assignment, index] + procs_cmd.loc[idx, "cmd_line"] = cmd # type: ignore break if weak_matches: From 8c1305c7691a96b084f4f81dfaaa17f4ccad0533 Mon Sep 17 00:00:00 2001 From: ianhelle Date: Fri, 22 Nov 2024 12:32:38 +0100 Subject: [PATCH 7/8] removing stray notebook --- msticpy/vis/TimelineInteractive.ipynb | 974 -------------------------- 1 file changed, 974 deletions(-) delete mode 100644 msticpy/vis/TimelineInteractive.ipynb diff --git a/msticpy/vis/TimelineInteractive.ipynb b/msticpy/vis/TimelineInteractive.ipynb deleted file mode 100644 index 63299f17f..000000000 --- a/msticpy/vis/TimelineInteractive.ipynb +++ /dev/null @@ -1,974 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Requirements\n", - "\n", - "1. Add data sources independently\n", - " - [x] Choose from available dataframes (variables)\n", - " - [] prompt for read from file?\n", - " - [ ] Select/edit time range from data\n", - " - [ ] Select time column to use\n", - " - [ ] Select properties for hover\n", - " - [ ] Enter series name\n", - " - [ ] Enter series color\n", - " - [ ] Select order (vertical position)\n", - "\n", - "2. Add annotation:\n", - " - [] Select time\n", - " - [] Choose from event?\n", - " - [] Add title\n", - " - [] Add text\n", - " - [] Tie to data set\n", - " - [] which data set should the annotation reference\n", - " - [] set display offset (to avoid overlaps)\n", - " - [] can we do this dynamically (keep track of what text has gone where)\n", - " - [] combine with \n", - " - [] build into DF\n", - "\n", - "3. Global\n", - " - Save settings\n", - " - Set\n", - " - font size\n", - " - width, height\n", - " - default color\n", - " - theme" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "from datetime import datetime\n", - "from functools import singledispatchmethod\n", - "from typing import Any, Dict, Iterable, List, Optional\n", - "\n", - "\n", - "import attr\n", - "from bokeh.models import LayoutDOM\n", - "import pandas as pd\n", - "from msticpy.vis.timeline import display_timeline\n", - "from msticpy.analysis.observationlist import Observations, Observation\n", - "from msticpy.common.timespan import TimeSpan\n", - "\n", - "\n", - "@attr.s(auto_attribs=True)\n", - "class DataSet:\n", - " data: pd.DataFrame\n", - " time_column: str\n", - " series_name: str\n", - " source_columns: attr.Factory(list)\n", - " color: Optional[str] = None\n", - " glyph: str = \"circle\"\n", - "\n", - "\n", - "\n", - "\n", - "class CompositeTimeline:\n", - "\n", - " _TIME_COLUMNS = (\"TimeGenerated\", \"EventTime\", \"timestamp\")\n", - "\n", - " def __init__(\n", - " self,\n", - " data: Optional[pd.DataFrame] = None,\n", - " series_name: Optional[str] = None,\n", - " data_sets: Optional[Dict[str, Any]] = None,\n", - " **kwargs,\n", - " ):\n", - " self._data_sets = Observations()\n", - " self._observations = Observations()\n", - " self._timeline_plot: Optional[LayoutDOM] = None\n", - "\n", - " if data:\n", - " self.add_data(data, series_name, **kwargs)\n", - " if data_sets:\n", - " for ds_series_name, ds_series_attrs in data_sets.items():\n", - " if isinstance(ds_series_attrs, DataSet):\n", - " ds_series_attrs = attr.asdict(ds_series_attrs)\n", - " self.add_data(\n", - " data=ds_series_attrs[\"data\"],\n", - " series_name=ds_series_name,\n", - " time_column=ds_series_attrs[\"time_column\"],\n", - " source_columns=ds_series_attrs.get(\n", - " \"time_column\",\n", - " self._determine_time_column(ds_series_attrs[\"data\"])\n", - " ),\n", - " color=ds_series_attrs.get(\"color\")\n", - " )\n", - "\n", - " @property\n", - " def data_set_names(self) -> List[str]:\n", - " return list(self._data_sets)\n", - " \n", - " def __getitem__(self, key: str) -> Observation:\n", - " return self._data_sets.get(key)\n", - "\n", - " def get_plot_params(self, caption: str) -> DataSet:\n", - " return self._data_sets[caption][\"additional_properties\"][\"plot_params\"]\n", - "\n", - " @singledispatchmethod\n", - " def add_data(\n", - " self,\n", - " data: pd.DataFrame,\n", - " series_name: Optional[str],\n", - " time_column: Optional[str] = None,\n", - " source_columns: Optional[Iterable[str]] = None,\n", - " color: Optional[str] = None,\n", - " glyph: str = \"circle\",\n", - " group_by: Optional[str] = None,\n", - " ) -> None:\n", - " if not time_column:\n", - " time_column = self._determine_time_column(data)\n", - "\n", - " if not time_column:\n", - " raise ValueError(\n", - " \"No value for time_column could be found in the data. \"\n", - " \"Please specify the time column in the 'time_column' parameter.\"\n", - " )\n", - " if not group_by:\n", - " self._add_data_set(\n", - " series_name=series_name,\n", - " data=data,\n", - " time_column=time_column,\n", - " source_columns=source_columns,\n", - " color=color\n", - " )\n", - " else:\n", - " for group, data_group in data.groupby(group_by):\n", - " series_group_name = f\"{series_name} ({group})\"\n", - " self._add_data_set(\n", - " series_name=series_group_name,\n", - " data=data_group,\n", - " time_column=time_column,\n", - " source_columns=source_columns,\n", - " color=color\n", - " )\n", - "\n", - " @add_data.register\n", - " def _(\n", - " self,\n", - " data_set: DataSet,\n", - " ) -> None:\n", - " if not data_set.time_column:\n", - " data_set.time_column = self._determine_time_column(data_set.data)\n", - "\n", - " if not data_set.time_column:\n", - " raise ValueError(\n", - " \"No value for time_column could be found in the data. \"\n", - " \"Please specify the time column in the 'time_column' attribute.\"\n", - " )\n", - " self._datasets[data_set.series_name] = data_set\n", - "\n", - " def _add_data_set(self, series_name, data, time_column, source_columns, color):\n", - " new_data = Observation(\n", - " caption=series_name,\n", - " data=data,\n", - " data_type=\"DataFrame\",\n", - " time_span=TimeSpan(start=data[time_column].min(), end=data[time_column].max()),\n", - " time_column=time_column\n", - " )\n", - " new_data.additional_properties[\"plot_params\"] = DataSet(\n", - " data=data,\n", - " series_name=series_name,\n", - " source_columns=source_columns,\n", - " time_column=time_column,\n", - " color=color,\n", - " )\n", - " self._datasets[series_name] = new_data\n", - "\n", - " def add_annotation(\n", - " self,\n", - " caption: str,\n", - " description: str,\n", - " timestamp: datetime,\n", - " **kwargs,\n", - " ):\n", - " obs_kwargs = {\n", - " key: value for key, value in kwargs.items()\n", - " if key in Observation.all_fields()\n", - " }\n", - " self._observations.add_observation(\n", - " Observation(\n", - " caption=caption, description=description, timestamp=timestamp, **obs_kwargs\n", - " )\n", - " )\n", - "\n", - " def add_observations(self, observations: Iterable[Observation]):\n", - " for observation in observations:\n", - " self.add_observation(observation)\n", - "\n", - " def add_observation(\n", - " self,\n", - " observation: Observation,\n", - " **kwargs,\n", - " ) -> None:\n", - " self._observations.add_observation(observation, **kwargs)\n", - " observation_time = observation.timestamp or kwargs.get(\"timestamp\")\n", - " if not observation_time:\n", - " print(\"No time_stamp supplied, observation will not be plotted.\")\n", - " observation.timestamp = observation_time\n", - "\n", - " def display(self):\n", - "\n", - " # Add normal data series\n", - " timeline_dict = {series_name: attr.asdict(data_set) for series_name, data_set in self._datasets.items()}\n", - "\n", - " # Add observations\n", - " # TODO\n", - "\n", - " def _determine_time_column(self, data: pd.DataFrame) -> Optional[str]:\n", - " return next(iter(\n", - " col for col_lower, col in ((col.casefold(), col)\n", - " for col in data.columns)\n", - " if col_lower in self._TIME_COLUMNS\n", - " ), None)\n", - "\n", - " # def _obs_list_no_expand_to_df(self):\n", - " # \"\"\"Return observations as a DataFrame.\"\"\"\n", - " # obs_list = [\n", - " # attr.asdict(obs, filter=lambda attr, _: attr != \"data\")\n", - " # for obs in self.observation_list.values()\n", - " # if not obs.expand_data\n", - " # ]\n", - " # return pd.json_normalize(obs_list)\n", - "\n", - " # def _obs_list_data_to_df(self):\n", - " # obs_list = [\n", - " # attr.asdict(obs, filter=lambda attr, _: attr != \"data\")\n", - " # for obs in self.observation_list.values()\n", - " # if obs.expand_data\n", - " # ]\n", - " # obs_dfs = [\n", - " # obs.data[[obs.time_column]].rename(columns={obs.time_column, \"time_stamp\"})\n", - " # for obs in self.observation_list.values()\n", - " # if obs.expand_data\n", - " # ]\n", - "\n", - "\n", - "@attr.s(auto_attribs=True):\n", - "\n", - "class TLAnnotation\n", - "\n", - "class TLAnnotationsSet:\n", - "\n", - " def __init__(self):\n", - " self.annotations = List[TLAnnotation]\n", - " def add_annotation()\n" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "dict_keyiterator" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dt = {1: 2, 3: 4}\n", - "type(iter(dt))" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['__class__',\n", - " '__delattr__',\n", - " '__dict__',\n", - " '__dir__',\n", - " '__doc__',\n", - " '__eq__',\n", - " '__format__',\n", - " '__ge__',\n", - " '__getattribute__',\n", - " '__gt__',\n", - " '__hash__',\n", - " '__init__',\n", - " '__init_subclass__',\n", - " '__le__',\n", - " '__lt__',\n", - " '__module__',\n", - " '__ne__',\n", - " '__new__',\n", - " '__reduce__',\n", - " '__reduce_ex__',\n", - " '__repr__',\n", - " '__setattr__',\n", - " '__sizeof__',\n", - " '__str__',\n", - " '__subclasshook__',\n", - " '__weakref__',\n", - " '_colors']" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from bokeh.colors import groups\n", - "color_groups = groups.__all__\n", - "color_groups[0]\n", - "dir(groups.black)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "49f934ad901b4aa6bf2c4614061e6305", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Button(description='Help', style=ButtonStyle(), tooltip='Confirms updates to the settings changes')" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import ipywidgets as widgets\n", - "\n", - "btn = widgets.Button(description=\"Help\", tooltip=\"Confirms updates to the settings changes\")\n", - "btn" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "39f7d87ec1804454bbc250ff77bc9fc2", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Dropdown(description='Help', options=('one', 'two'), value='one')" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "widgets.Dropdown(description=\"Help\", options=[\"one\", \"two\"], tooltip=\"Confirms updates to the settings changes\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# UI" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "import re\n", - "\n", - "import pandas as pd\n", - "\n", - "def get_global_dfs(pattern: str = None):\n", - " \n", - " return [\n", - " name for name in globals()\n", - " if (not pattern or re.match(pattern, name))\n", - " and isinstance(globals()[name], pd.DataFrame)\n", - " and not name.startswith(\"_\")\n", - " ]\n", - "\n", - "var1 = pd.DataFrame()\n", - "xxx = pd.DataFrame()\n", - "\n", - "def get_df_var(var_name: str) -> pd.DataFrame:\n", - " if var_name in globals():\n", - " return globals()[var_name]" - ] - }, - { - "cell_type": "code", - "execution_count": 79, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "c09e3395dc2d4049a835385565b1d906", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "AppLayout(children=(HTML(value='

Timeline builder

', layout=Layout(grid_area='header')), Label(value='T…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "import ipywidgets as widgets\n", - "\n", - "\n", - "\n", - "from msticpy.config.file_browser import FileBrowser\n", - "\n", - "\n", - "DESC_WIDTH = {\"style\": {\"description_width\": \"120px\"}}\n", - "\n", - "WIDTH_80 = {\"layout\": widgets.Layout(width=\"80%\")}\n", - "\n", - "def border_layout(width=\"95%\"):\n", - " \"\"\"Return border widget layout.\"\"\"\n", - " return {\n", - " \"layout\": widgets.Layout(\n", - " **{\n", - " \"width\": width,\n", - " \"border\": \"solid gray 1px\",\n", - " \"margin\": \"1pt\",\n", - " \"padding\": \"5pt\",\n", - " }\n", - " )\n", - " }\n", - "\n", - "\n", - "\n", - "\n", - "btn_get_file_name = widgets.Button(description=\"Select file...\", tooltip=\"Browse for DataFrame\")\n", - "FileBrowser()\n", - "\n", - "from msticpy.nbtools.nbwidgets import QueryTime\n", - "\n", - "c_timeline = CompositeTimeline()\n", - "\n", - "\n", - "\n", - "sel_ds = widgets.Select(description=\"Datasets\", options=c_timeline.data_set_names)\n", - "\n", - "# Basic data set edit controls\n", - "txt_ds_caption = widgets.Text(description=\"Series name\", **DESC_WIDTH)\n", - "sel_ds_time_column = widgets.Select(description=\"Time column\", **DESC_WIDTH)\n", - "txt_ds_color = widgets.Text(description=\"Series color\", **DESC_WIDTH)\n", - "sel_ds_groupby = widgets.Select(description=\"Group by\", **DESC_WIDTH)\n", - "msel_ds_tooltips = widgets.SelectMultiple(description=\"Tooltip cols\", **DESC_WIDTH)\n", - "\n", - "\n", - "btn_ds_del = widgets.Button(description=\"Delete\")\n", - "btn_ds_apply = widgets.Button(description=\"Save\")\n", - "vb_ds_edit1 = widgets.VBox([txt_ds_caption, sel_ds_time_column, txt_ds_color])\n", - "vb_ds_edit2 = widgets.VBox([sel_ds_groupby, msel_ds_tooltips])\n", - "hb_ds_edit = widgets.HBox([vb_ds_edit1, vb_ds_edit2, btn_ds_apply], **border_layout(\"70%\"))\n", - "vb_ds_list = widgets.VBox([sel_ds, btn_ds_del], **border_layout(\"25%\"))\n", - "hb_data_set = widgets.HBox([vb_ds_list, hb_ds_edit])\n", - "\n", - "sel_obs = widgets.Select(description=\"Observations\")\n", - "# Observation edit controls\n", - "txt_obs_caption = widgets.Text(description=\"Caption\", **DESC_WIDTH, **WIDTH_80)\n", - "txt_obs_desc = widgets.Textarea(description=\"Text\", **DESC_WIDTH, **WIDTH_80)\n", - "dt_obs_time = widgets.DatePicker(description=\"Timestamp\")\n", - "btn_obs_new = widgets.Button(description=\"New\")\n", - "lbl_obs_sliders = widgets.Label(value=\"Annotation positioning\")\n", - "SLIDER_PARAMS = dict(\n", - " value=0,\n", - " min=-10,\n", - " max=10,\n", - " step=1,\n", - " readout=True,\n", - " continuous_update=False,\n", - " readout_format='d',\n", - " orientation='horizontal',\n", - ")\n", - "int_obs_hoff = widgets.IntSlider(\n", - " description='H Offset:',\n", - " **SLIDER_PARAMS,\n", - ")\n", - "int_obs_voff = widgets.IntSlider(\n", - " description='V Offset:',\n", - " **SLIDER_PARAMS,\n", - ")\n", - "btn_obs_del = widgets.Button(description=\"Delete\")\n", - "btn_obs_save = widgets.Button(description=\"Save\")\n", - "btn_obs_new = widgets.Button(description=\"Save new\")\n", - "btn_obs_clear = widgets.Button(description=\"Clear\")\n", - "hb_obs_buttons = widgets.HBox([btn_obs_save, btn_obs_new, btn_obs_clear])\n", - "vb_obs_edit1 = widgets.VBox([txt_obs_caption, txt_obs_desc, dt_obs_time, hb_obs_buttons], layout=widgets.Layout(width=\"60%\"))\n", - "vb_obs_edit2 = widgets.VBox([lbl_obs_sliders, int_obs_hoff, int_obs_voff])\n", - "hb_obs_edit = widgets.HBox([vb_obs_edit1, vb_obs_edit2], **border_layout(\"65%\"))\n", - "vb_obs_list = widgets.VBox([sel_obs, btn_obs_del], **border_layout(\"25%\"))\n", - "hb_obs = widgets.HBox([vb_obs_list, hb_obs_edit])\n", - "\n", - "\n", - "\n", - "# Add data frame\n", - "sel_add_var_name = widgets.Select(\n", - " description=\"Dataframe variables\",\n", - " options=get_global_dfs(),\n", - " **DESC_WIDTH,\n", - ")\n", - "btn_add_add = widgets.Button(description=\"Add DataFrame\", tooltip=\"Add DataFrame to Datasets.\")\n", - "btn_add_refresh = widgets.Button(description=\"Refresh vars\", tooltip=\"Add DataFrame to Datasets.\")\n", - "hb_add_dataset = widgets.HBox([sel_add_var_name, btn_add_refresh, btn_add_add])\n", - "\n", - "# Filter expression\n", - "txt_df_filter = widgets.Textarea(description=\"Pandas query\", layout=widgets.Layout(height=\"200px\", width=\"50%\"))\n", - "btn_apply_query = widgets.Button(description=\"Apply\", tooltip=\"Apply query to data.\")\n", - "vb_query = widgets.VBox([txt_df_filter, btn_apply_query])\n", - "\n", - "# Time range\n", - "qt_select_time_range = QueryTime(description=\"Filter time range of data\")\n", - "btn_apply_time_range = widgets.Button(description=\"Save\", tooltip=\"Apply selected time range to data.\")\n", - "vb_time_range = widgets.VBox([qt_select_time_range.layout, btn_apply_time_range])\n", - "\n", - "\n", - "accd_filter = widgets.Accordion(children=[vb_time_range, vb_query])\n", - "accd_filter.set_title(0, \"Filter time range\")\n", - "accd_filter.set_title(1, \"Filter data\")\n", - "accd_filter.selected_index = None\n", - "vb_data_set_edit = widgets.VBox([hb_data_set, accd_filter])\n", - "\n", - "tab_ds_obs = widgets.Tab(children=[vb_data_set_edit, hb_obs, hb_add_dataset])\n", - "tab_ds_obs.set_title(0, \"Loaded data sets\")\n", - "tab_ds_obs.set_title(1, \"Annotations\")\n", - "tab_ds_obs.set_title(2, \"Add data sets\")\n", - "\n", - "lbl_status = widgets.Label(value=\"Test status\")\n", - "\n", - "widgets.Layout()\n", - "html_title = widgets.HTML(value=\"

Timeline builder

\", style={\"text-align\": \"center\"})\n", - "app_layout = widgets.AppLayout(\n", - " header=html_title,\n", - " center=tab_ds_obs,\n", - " footer=lbl_status,\n", - " pane_heights=[1, \"400px\", 1]\n", - ")\n", - "\n", - "app_layout" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "## Event handlers\n", - "\n", - "def _refresh_df_vars(btn):\n", - " del btn\n", - " sel_add_var_name.options=get_global_dfs()\n", - "\n", - "btn_add_refresh.on_click(_refresh_df_vars)\n", - "\n", - "def _select_dataset(change):\n", - " df_key = change.get(\"new\")\n", - " data = _get_data_for_key(df_key)\n", - " _update_groupby(data)\n", - " _update_timecolumn(data)\n", - " _update_status(data)\n", - "\n", - "sel_ds.observe(_select_dataset, names=\"value\")\n", - "\n", - "def _save_ds_changes(btn):\n", - " \"\"\"Click handler for save changes to current set.\"\"\"\n", - " if not txt_ds_caption.value:\n", - "\n", - "\n", - "def _update_groupby(data):\n", - " sel_ds_groupby.options = [\n", - " \"None\",\n", - " *(sorted(data.columns))\n", - " ]\n", - "\n", - "def _update_timecolumn(data):\n", - " sel_ds_time_column.options = [\n", - " \"None\",\n", - " *(sorted(data.select_dtypes(include=[\"datetime\", \"datetimetz\"]).columns))\n", - " ]\n", - "\n", - "def _update_status(data):\n", - " # TODO apply filter and timespan\n", - " time_col = sel_ds_time_column.value\n", - " status = (\n", - " f\"Selected records: {len(data)} \"\n", - " f\"Start time: {data[time_col].min().isoformat()} \"\n", - " f\"End time: {data[time_col].max().isoformat()} \"\n", - " )\n", - "\n", - "# stubs\n", - "def _get_data_for_key(key):\n", - " return mde_df\n", - "\n", - "def _update_dataset(caption, time_column, group_by, color, source_columns):\n", - " return True\n" - ] - }, - { - "cell_type": "code", - "execution_count": 73, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
TimestampProcessCreationTimeInitiatingProcessCreationTimeInitiatingProcessParentCreationTime
02021-12-15 23:00:03.449464400+00:002021-12-15 23:00:03.449464400+00:002021-11-11 03:26:31.574305+00:002021-11-11 03:26:28.153467700+00:00
12021-12-15 23:00:03.449833900+00:002021-12-15 23:00:03.449833900+00:002021-12-15 23:00:03.038042200+00:002021-11-11 03:26:31.574305+00:00
22021-12-15 23:11:27.173362800+00:002021-12-15 23:11:27.173362800+00:002021-11-11 03:26:32.145838600+00:002021-11-11 03:26:28.153467700+00:00
32021-12-15 23:11:37.869622100+00:002021-12-15 23:11:37.869622100+00:002021-12-15 23:11:26.473328+00:002021-11-11 03:26:32.145838600+00:00
42021-12-15 23:11:37.870024200+00:002021-12-15 23:11:37.870024200+00:002021-12-15 23:11:37.777611600+00:002021-12-15 23:11:26.473328+00:00
...............
1952021-12-16 03:31:11.600695600+00:002021-12-16 03:31:11.600695600+00:002021-12-16 03:28:10.765459700+00:002021-12-16 03:28:07.042663300+00:00
1962021-12-16 03:31:12.486054600+00:002021-12-16 03:31:12.486054600+00:002021-12-16 03:28:07.042663300+00:002021-12-16 03:28:06.996438500+00:00
1972021-12-16 03:32:11.149138600+00:002021-12-16 03:32:11.149138600+00:002021-12-16 03:28:10.930604400+00:002021-12-16 03:28:07.042663300+00:00
1982021-12-16 03:32:14.034770600+00:002021-12-16 03:32:14.034770600+00:002021-12-16 03:28:07.231891200+00:002021-12-16 03:28:07.042663300+00:00
1992021-12-16 03:32:14.040591700+00:002021-12-16 03:32:14.040591700+00:002021-12-16 03:28:07.042663300+00:002021-12-16 03:28:06.996438500+00:00
\n", - "

200 rows × 4 columns

\n", - "
" - ], - "text/plain": [ - " Timestamp ProcessCreationTime \\\n", - "0 2021-12-15 23:00:03.449464400+00:00 2021-12-15 23:00:03.449464400+00:00 \n", - "1 2021-12-15 23:00:03.449833900+00:00 2021-12-15 23:00:03.449833900+00:00 \n", - "2 2021-12-15 23:11:27.173362800+00:00 2021-12-15 23:11:27.173362800+00:00 \n", - "3 2021-12-15 23:11:37.869622100+00:00 2021-12-15 23:11:37.869622100+00:00 \n", - "4 2021-12-15 23:11:37.870024200+00:00 2021-12-15 23:11:37.870024200+00:00 \n", - ".. ... ... \n", - "195 2021-12-16 03:31:11.600695600+00:00 2021-12-16 03:31:11.600695600+00:00 \n", - "196 2021-12-16 03:31:12.486054600+00:00 2021-12-16 03:31:12.486054600+00:00 \n", - "197 2021-12-16 03:32:11.149138600+00:00 2021-12-16 03:32:11.149138600+00:00 \n", - "198 2021-12-16 03:32:14.034770600+00:00 2021-12-16 03:32:14.034770600+00:00 \n", - "199 2021-12-16 03:32:14.040591700+00:00 2021-12-16 03:32:14.040591700+00:00 \n", - "\n", - " InitiatingProcessCreationTime InitiatingProcessParentCreationTime \n", - "0 2021-11-11 03:26:31.574305+00:00 2021-11-11 03:26:28.153467700+00:00 \n", - "1 2021-12-15 23:00:03.038042200+00:00 2021-11-11 03:26:31.574305+00:00 \n", - "2 2021-11-11 03:26:32.145838600+00:00 2021-11-11 03:26:28.153467700+00:00 \n", - "3 2021-12-15 23:11:26.473328+00:00 2021-11-11 03:26:32.145838600+00:00 \n", - "4 2021-12-15 23:11:37.777611600+00:00 2021-12-15 23:11:26.473328+00:00 \n", - ".. ... ... \n", - "195 2021-12-16 03:28:10.765459700+00:00 2021-12-16 03:28:07.042663300+00:00 \n", - "196 2021-12-16 03:28:07.042663300+00:00 2021-12-16 03:28:06.996438500+00:00 \n", - "197 2021-12-16 03:28:10.930604400+00:00 2021-12-16 03:28:07.042663300+00:00 \n", - "198 2021-12-16 03:28:07.231891200+00:00 2021-12-16 03:28:07.042663300+00:00 \n", - "199 2021-12-16 03:28:07.042663300+00:00 2021-12-16 03:28:06.996438500+00:00 \n", - "\n", - "[200 rows x 4 columns]" - ] - }, - "execution_count": 73, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "mde_df.select_dtypes(include=[\"datetime\", \"datetimetz\"])\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def get_matching_vars\n", - "combo_df_var_name.observe(get_matching_vars, names=\"value\")" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " Loading BokehJS ...\n", - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n const force = true;\n\n if (typeof root._bokeh_onload_callbacks === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n const JS_MIME_TYPE = 'application/javascript';\n const HTML_MIME_TYPE = 'text/html';\n const EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n const CLASS_NAME = 'output_bokeh rendered_html';\n\n /**\n * Render data to the DOM node\n */\n function render(props, node) {\n const script = document.createElement(\"script\");\n node.appendChild(script);\n }\n\n /**\n * Handle when an output is cleared or removed\n */\n function handleClearOutput(event, handle) {\n const cell = handle.cell;\n\n const id = cell.output_area._bokeh_element_id;\n const server_id = cell.output_area._bokeh_server_id;\n // Clean up Bokeh references\n if (id != null && id in Bokeh.index) {\n Bokeh.index[id].model.document.clear();\n delete Bokeh.index[id];\n }\n\n if (server_id !== undefined) {\n // Clean up Bokeh references\n const cmd_clean = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n cell.notebook.kernel.execute(cmd_clean, {\n iopub: {\n output: function(msg) {\n const id = msg.content.text.trim();\n if (id in Bokeh.index) {\n Bokeh.index[id].model.document.clear();\n delete Bokeh.index[id];\n }\n }\n }\n });\n // Destroy server and session\n const cmd_destroy = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n cell.notebook.kernel.execute(cmd_destroy);\n }\n }\n\n /**\n * Handle when a new output is added\n */\n function handleAddOutput(event, handle) {\n const output_area = handle.output_area;\n const output = handle.output;\n\n // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n if ((output.output_type != \"display_data\") || (!Object.prototype.hasOwnProperty.call(output.data, EXEC_MIME_TYPE))) {\n return\n }\n\n const toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n\n if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n toinsert[toinsert.length - 1].firstChild.textContent = output.data[JS_MIME_TYPE];\n // store reference to embed id on output_area\n output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n }\n if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n const bk_div = document.createElement(\"div\");\n bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n const script_attrs = bk_div.children[0].attributes;\n for (let i = 0; i < script_attrs.length; i++) {\n toinsert[toinsert.length - 1].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n toinsert[toinsert.length - 1].firstChild.textContent = bk_div.children[0].textContent\n }\n // store reference to server id on output_area\n output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n }\n }\n\n function register_renderer(events, OutputArea) {\n\n function append_mime(data, metadata, element) {\n // create a DOM node to render to\n const toinsert = this.create_output_subarea(\n metadata,\n CLASS_NAME,\n EXEC_MIME_TYPE\n );\n this.keyboard_manager.register_events(toinsert);\n // Render to node\n const props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n render(props, toinsert[toinsert.length - 1]);\n element.append(toinsert);\n return toinsert\n }\n\n /* Handle when an output is cleared or removed */\n events.on('clear_output.CodeCell', handleClearOutput);\n events.on('delete.Cell', handleClearOutput);\n\n /* Handle when a new output is added */\n events.on('output_added.OutputArea', handleAddOutput);\n\n /**\n * Register the mime type and append_mime function with output_area\n */\n OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n /* Is output safe? */\n safe: true,\n /* Index of renderer in `output_area.display_order` */\n index: 0\n });\n }\n\n // register the mime type if in Jupyter Notebook environment and previously unregistered\n if (root.Jupyter !== undefined) {\n const events = require('base/js/events');\n const OutputArea = require('notebook/js/outputarea').OutputArea;\n\n if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n register_renderer(events, OutputArea);\n }\n }\n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n const NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"
    \\n\"+\n \"
  • re-rerun `output_notebook()` to attempt to load from CDN again, or
    • \\n\"+\n \"
  • use INLINE resources instead, as so:
    • \\n\"+\n \"
\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n const el = document.getElementById(\"1002\");\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) {\n if (callback != null)\n callback();\n });\n } finally {\n delete root._bokeh_onload_callbacks\n }\n console.debug(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(css_urls, js_urls, callback) {\n if (css_urls == null) css_urls = [];\n if (js_urls == null) js_urls = [];\n\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.debug(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.debug(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = css_urls.length + js_urls.length;\n\n function on_load() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: all BokehJS libraries/stylesheets loaded\");\n run_callbacks()\n }\n }\n\n function on_error(url) {\n console.error(\"failed to load \" + url);\n }\n\n for (let i = 0; i < css_urls.length; i++) {\n const url = css_urls[i];\n const element = document.createElement(\"link\");\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.rel = \"stylesheet\";\n element.type = \"text/css\";\n element.href = url;\n console.debug(\"Bokeh: injecting link tag for BokehJS stylesheet: \", url);\n document.body.appendChild(element);\n }\n\n for (let i = 0; i < js_urls.length; i++) {\n const url = js_urls[i];\n const element = document.createElement('script');\n element.onload = on_load;\n element.onerror = on_error.bind(null, url);\n element.async = false;\n element.src = url;\n console.debug(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.head.appendChild(element);\n }\n };\n\n function inject_raw_css(css) {\n const element = document.createElement(\"style\");\n element.appendChild(document.createTextNode(css));\n document.body.appendChild(element);\n }\n\n \n const js_urls = [\"https://cdn.bokeh.org/bokeh/release/bokeh-2.4.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-gl-2.4.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-widgets-2.4.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-tables-2.4.2.min.js\", \"https://cdn.bokeh.org/bokeh/release/bokeh-mathjax-2.4.2.min.js\"];\n const css_urls = [];\n \n\n const inline_js = [\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n function(Bokeh) {\n \n \n }\n ];\n\n function run_inline_js() {\n \n if (root.Bokeh !== undefined || force === true) {\n \n for (let i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }\n if (force === true) {\n display_loaded();\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n const cell = $(document.getElementById(\"1002\")).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n\n }\n\n if (root._bokeh_is_loading === 0) {\n console.debug(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(css_urls, js_urls, function() {\n console.debug(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));", - "application/vnd.bokehjs_load.v0+json": "" - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "
\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/javascript": "(function(root) {\n function embed_document(root) {\n \n const docs_json = {\"d6e21d0a-d676-473e-948e-daea2d8fbf3f\":{\"defs\":[],\"roots\":{\"references\":[{\"attributes\":{\"children\":[{\"id\":\"1005\"},{\"id\":\"1037\"}]},\"id\":\"1075\",\"type\":\"Column\"},{\"attributes\":{\"num_minor_ticks\":5,\"tickers\":[{\"id\":\"1101\"},{\"id\":\"1102\"},{\"id\":\"1103\"},{\"id\":\"1104\"},{\"id\":\"1105\"},{\"id\":\"1106\"},{\"id\":\"1107\"},{\"id\":\"1108\"},{\"id\":\"1109\"},{\"id\":\"1110\"},{\"id\":\"1111\"},{\"id\":\"1112\"}]},\"id\":\"1049\",\"type\":\"DatetimeTicker\"},{\"attributes\":{\"active_multi\":{\"id\":\"1062\"},\"tools\":[{\"id\":\"1062\"}]},\"id\":\"1052\",\"type\":\"Toolbar\"},{\"attributes\":{\"align\":\"right\",\"coordinates\":null,\"group\":null,\"text\":\"Drag the middle or edges of the selection box to change the range in the main chart\",\"text_font_size\":\"10px\"},\"id\":\"1053\",\"type\":\"Title\"},{\"attributes\":{\"months\":[0,4,8]},\"id\":\"1097\",\"type\":\"MonthsTicker\"},{\"attributes\":{\"axis\":{\"id\":\"1048\"},\"coordinates\":null,\"group\":null,\"ticker\":null},\"id\":\"1051\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1085\",\"type\":\"UnionRenderers\"},{\"attributes\":{},\"id\":\"1086\",\"type\":\"Selection\"},{\"attributes\":{\"coordinates\":null,\"group\":null,\"text\":\"Range Selector\"},\"id\":\"1038\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"1082\",\"type\":\"AllLabels\"},{\"attributes\":{\"days\":[1,8,15,22]},\"id\":\"1093\",\"type\":\"DaysTicker\"},{\"attributes\":{},\"id\":\"1080\",\"type\":\"AllLabels\"},{\"attributes\":{},\"id\":\"1044\",\"type\":\"LinearScale\"},{\"attributes\":{\"months\":[0,6]},\"id\":\"1098\",\"type\":\"MonthsTicker\"},{\"attributes\":{},\"id\":\"1046\",\"type\":\"LinearScale\"},{\"attributes\":{\"coordinates\":null,\"formatter\":{\"id\":\"1054\"},\"group\":null,\"major_label_policy\":{\"id\":\"1084\"},\"ticker\":{\"id\":\"1049\"}},\"id\":\"1048\",\"type\":\"DatetimeAxis\"},{\"attributes\":{\"months\":[0,1,2,3,4,5,6,7,8,9,10,11]},\"id\":\"1095\",\"type\":\"MonthsTicker\"},{\"attributes\":{},\"id\":\"1042\",\"type\":\"DataRange1d\"},{\"attributes\":{\"days\":[1,15]},\"id\":\"1094\",\"type\":\"DaysTicker\"},{\"attributes\":{\"months\":[0,2,4,6,8,10]},\"id\":\"1096\",\"type\":\"MonthsTicker\"},{\"attributes\":{\"days\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]},\"id\":\"1091\",\"type\":\"DaysTicker\"},{\"attributes\":{},\"id\":\"1079\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"months\":[0,6]},\"id\":\"1111\",\"type\":\"MonthsTicker\"},{\"attributes\":{\"days\":[1,4,7,10,13,16,19,22,25,28]},\"id\":\"1092\",\"type\":\"DaysTicker\"},{\"attributes\":{\"fill_color\":{\"value\":\"navy\"},\"hatch_color\":{\"value\":\"navy\"},\"line_color\":{\"value\":\"navy\"},\"x\":{\"field\":\"Timestamp\"},\"y\":{\"field\":\"y_index\"}},\"id\":\"1057\",\"type\":\"Circle\"},{\"attributes\":{},\"id\":\"1112\",\"type\":\"YearsTicker\"},{\"attributes\":{\"months\":[0,4,8]},\"id\":\"1110\",\"type\":\"MonthsTicker\"},{\"attributes\":{},\"id\":\"1099\",\"type\":\"YearsTicker\"},{\"attributes\":{\"source\":{\"id\":\"1003\"}},\"id\":\"1074\",\"type\":\"CDSView\"},{\"attributes\":{},\"id\":\"1026\",\"type\":\"ResetTool\"},{\"attributes\":{},\"id\":\"1021\",\"type\":\"BasicTicker\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1024\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"coordinates\":null,\"formatter\":{\"id\":\"1079\"},\"group\":null,\"major_label_policy\":{\"id\":\"1080\"},\"ticker\":{\"id\":\"1021\"},\"visible\":false},\"id\":\"1020\",\"type\":\"LinearAxis\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"navy\"},\"hatch_alpha\":{\"value\":0.1},\"hatch_color\":{\"value\":\"navy\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"navy\"},\"marker\":{\"value\":\"diamond\"},\"size\":{\"value\":10},\"x\":{\"field\":\"Timestamp\"},\"y\":{\"field\":\"y_index\"}},\"id\":\"1071\",\"type\":\"Scatter\"},{\"attributes\":{\"data\":{\"Timestamp\":{\"__ndarray__\":\"bZevLwXcd0JUna8vBdx3QstVnNYF3HdC9Nk42QXcd0Ji4DjZBdx3QuleG9sF3HdCpoEb2wXcd0LVmE1hB9x3QlBdZGMH3HdCFJSeYwfcd0KH9ASeCNx3QrhGaxAJ3HdCO01rEAncd0KHjv5JCdx3Qqp5RTYK3HdCUh5A0Arcd0LFRiPKC9x3QvyLrMwL3HdCfZGszAvcd0KR5ejOC9x3QhuT9M4L3HdCI1kd0Qvcd0II3PN5DNx3QnHj83kM3HdCWMGQewzcd0LbX757DNx3QlilvnsM3HdCYlAIPw7cd0KHhgg/Dtx3QhfNN8kO3HdCh9I3yQ7cd0KWQ1P2D9x3QhmS6a0R3HdCpFL1TxLcd0KwFpxSEtx3QvwbnFIS3HdCTsaNVBLcd0IX+41UEtx3Qt/fgeoS3HdC152i6xLcd0LDVe7rEtx3QmJm7usS3HdCkVvK7hLcd0ItwlXvEtx3QpzsLvIS3HdCrLzp+xLcd0I/acn8Etx3Qt0mUAUT3HdCouvBBRPcd0IExhQME9x3QsMpygwT3HdC2y/6HhPcd0JvstFhE9x3Qjk81GET3HdCuHB0gxPcd0JUo5+fE9x3Qo2pn58T3HdCVK2fnxPcd0KBs5+fE9x3Qvy3n58T3HdCnr2fnxPcd0Izz5+fE9x3QiXUn58T3HdCuvF4oxPcd0KHptnGE9x3Qjfp48YT3HdCEuOY9hPcd0JvDmL3E9x3Ql4MpY8U3HdC03uljxTcd0L+qKWPFNx3Qjsjpo8U3HdCrCamjxTcd0KLRqaPFNx3Qgxepo8U3HdCNairjxTcd0KJpa6PFNx3QnXBv48U3HdCZiLAjxTcd0LwT8CPFNx3Qn3twI8U3HdCBsHVjxTcd0KuzdaPFNx3Qucp148U3HdCP2XfjxTcd0LufN+PFNx3QvaM348U3HdCFDzijxTcd0KuA+OPFNx3QjF2448U3HdCmkfkjxTcd0KRVeSPFNx3Qvxh5I8U3HdCf2jkjxTcd0LwbeSPFNx3Qp6B5I8U3HdCpofkjxTcd0Lsi+SPFNx3QhBE5Y8U3HdC7FnljxTcd0L4XeWPFNx3QmJi5Y8U3HdCnGbljxTcd0KsauWPFNx3Qi9v5Y8U3HdCbXXljxTcd0LfeeWPFNx3QhB+5Y8U3HdCyYLljxTcd0I7h+WPFNx3QiuN5Y8U3HdClpHljxTcd0JzluWPFNx3Qkib5Y8U3HdCmp/ljxTcd0J1o+WPFNx3Qomn5Y8U3HdClqvljxTcd0ICseWPFNx3QkjB5Y8U3HdCWMfljxTcd0I/zeWPFNx3Qr7R5Y8U3HdCUNvljxTcd0LP3+WPFNx3QhLl5Y8U3HdC8urljxTcd0Lu7uWPFNx3Qkzz5Y8U3HdCXP/ljxTcd0JaGuaPFNx3Qk4e5o8U3HdCGSLmjxTcd0JOJuaPFNx3QhQq5o8U3HdCqi3mjxTcd0L0MeaPFNx3Qpo15o8U3HdCujnmjxTcd0J7cOaPFNx3Qht15o8U3HdC2X7mjxTcd0IKg+aPFNx3Qh+H5o8U3HdCUozmjxTcd0K4nOaPFNx3QrCg5o8U3HdCoLTmjxTcd0J1veaPFNx3QnPE5o8U3HdCqMrmjxTcd0KNz+aPFNx3QtfT5o8U3HdCPdjmjxTcd0Ko3OaPFNx3Qrbt5o8U3HdC5/HmjxTcd0IA9uaPFNx3Qgz65o8U3HdCTv7mjxTcd0JSAuePFNx3QikQ548U3HdCbRnnjxTcd0K8JuePFNx3Qkwr548U3HdCMTbnjxTcd0IERuePFNx3QotK548U3HdCvm3njxTcd0KJceePFNx3Qlh9548U3HdCiYHnjxTcd0J9h+ePFNx3QsWg548U3HdCVLfnjxTcd0IvL+mPFNx3QuwL7o8U3HdCNfrujxTcd0IrXRSQFNx3QgxmGpAU3HdCUMGXkBTcd0KHBtGQFNx3Qh8n0ZAU3HdCz/VBkRTcd0IxWNqkFNx3QkTr2qQU3HdCQrz9pBTcd0LsA/6kFNx3QgyUEKUU3HdCVkwzpRTcd0IhSJalFNx3QmB7lqUU3HdCh/JIsxTcd0J9AUmzFNx3Qs0oSbMU3HdCHwtnsxTcd0LdYJ6zFNx3QjXS8MEU3HdCUiylwhTcd0J1iaXCFNx3Qg==\",\"dtype\":\"float64\",\"order\":\"little\",\"shape\":[200]},\"index\":[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199],\"y_index\":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},\"selected\":{\"id\":\"1086\"},\"selection_policy\":{\"id\":\"1085\"}},\"id\":\"1003\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"mantissas\":[1,2,5],\"max_interval\":500.0,\"num_minor_ticks\":0},\"id\":\"1101\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"navy\"},\"hatch_alpha\":{\"value\":0.5},\"hatch_color\":{\"value\":\"navy\"},\"line_alpha\":{\"value\":0.5},\"line_color\":{\"value\":\"navy\"},\"marker\":{\"value\":\"diamond\"},\"size\":{\"value\":10},\"x\":{\"field\":\"Timestamp\"},\"y\":{\"field\":\"y_index\"}},\"id\":\"1070\",\"type\":\"Scatter\"},{\"attributes\":{\"months\":[0,1,2,3,4,5,6,7,8,9,10,11]},\"id\":\"1108\",\"type\":\"MonthsTicker\"},{\"attributes\":{\"start\":-1.0},\"id\":\"1010\",\"type\":\"Range1d\"},{\"attributes\":{\"coordinates\":null,\"data_source\":{\"id\":\"1003\"},\"glyph\":{\"id\":\"1070\"},\"group\":null,\"hover_glyph\":null,\"muted_glyph\":{\"id\":\"1072\"},\"nonselection_glyph\":{\"id\":\"1071\"},\"view\":{\"id\":\"1074\"}},\"id\":\"1073\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"days\":[1,4,7,10,13,16,19,22,25,28]},\"id\":\"1105\",\"type\":\"DaysTicker\"},{\"attributes\":{},\"id\":\"1084\",\"type\":\"AllLabels\"},{\"attributes\":{\"num_minor_ticks\":10,\"tickers\":[{\"id\":\"1088\"},{\"id\":\"1089\"},{\"id\":\"1090\"},{\"id\":\"1091\"},{\"id\":\"1092\"},{\"id\":\"1093\"},{\"id\":\"1094\"},{\"id\":\"1095\"},{\"id\":\"1096\"},{\"id\":\"1097\"},{\"id\":\"1098\"},{\"id\":\"1099\"}]},\"id\":\"1017\",\"type\":\"DatetimeTicker\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.1},\"fill_color\":{\"value\":\"navy\"},\"hatch_alpha\":{\"value\":0.1},\"hatch_color\":{\"value\":\"navy\"},\"line_alpha\":{\"value\":0.1},\"line_color\":{\"value\":\"navy\"},\"x\":{\"field\":\"Timestamp\"},\"y\":{\"field\":\"y_index\"}},\"id\":\"1058\",\"type\":\"Circle\"},{\"attributes\":{\"days\":[1,15]},\"id\":\"1107\",\"type\":\"DaysTicker\"},{\"attributes\":{\"coordinates\":null,\"data_source\":{\"id\":\"1003\"},\"glyph\":{\"id\":\"1057\"},\"group\":null,\"hover_glyph\":null,\"muted_glyph\":{\"id\":\"1059\"},\"nonselection_glyph\":{\"id\":\"1058\"},\"view\":{\"id\":\"1061\"}},\"id\":\"1060\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"source\":{\"id\":\"1003\"}},\"id\":\"1061\",\"type\":\"CDSView\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.2},\"fill_color\":{\"value\":\"navy\"},\"hatch_alpha\":{\"value\":0.2},\"hatch_color\":{\"value\":\"navy\"},\"line_alpha\":{\"value\":0.2},\"line_color\":{\"value\":\"navy\"},\"x\":{\"field\":\"Timestamp\"},\"y\":{\"field\":\"y_index\"}},\"id\":\"1059\",\"type\":\"Circle\"},{\"attributes\":{},\"id\":\"1027\",\"type\":\"SaveTool\"},{\"attributes\":{\"end\":1639627983629.2607,\"start\":1639606753860.7954},\"id\":\"1040\",\"type\":\"Range1d\"},{\"attributes\":{\"days\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]},\"id\":\"1104\",\"type\":\"DaysTicker\"},{\"attributes\":{\"axis\":{\"id\":\"1020\"},\"coordinates\":null,\"dimension\":1,\"grid_line_color\":null,\"group\":null,\"ticker\":null},\"id\":\"1023\",\"type\":\"Grid\"},{\"attributes\":{\"days\":[\"%m-%d %H:%M\"],\"hours\":[\"%H:%M:%S\"],\"milliseconds\":[\"%H:%M:%S.%3N\"],\"minutes\":[\"%H:%M:%S\"],\"seconds\":[\"%H:%M:%S\"]},\"id\":\"1067\",\"type\":\"DatetimeTickFormatter\"},{\"attributes\":{\"below\":[{\"id\":\"1016\"}],\"center\":[{\"id\":\"1019\"},{\"id\":\"1023\"}],\"height\":300,\"left\":[{\"id\":\"1020\"}],\"min_border_left\":50,\"renderers\":[{\"id\":\"1073\"}],\"title\":{\"id\":\"1006\"},\"toolbar\":{\"id\":\"1030\"},\"width\":900,\"x_range\":{\"id\":\"1008\"},\"x_scale\":{\"id\":\"1012\"},\"y_range\":{\"id\":\"1010\"},\"y_scale\":{\"id\":\"1014\"}},\"id\":\"1005\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"base\":24,\"mantissas\":[1,2,4,6,8,12],\"max_interval\":43200000.0,\"min_interval\":3600000.0,\"num_minor_ticks\":0},\"id\":\"1103\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"days\":[\"%m-%d %H:%M\"],\"hours\":[\"%H:%M:%S\"],\"milliseconds\":[\"%H:%M:%S.%3N\"],\"minutes\":[\"%H:%M:%S\"],\"seconds\":[\"%H:%M:%S\"]},\"id\":\"1054\",\"type\":\"DatetimeTickFormatter\"},{\"attributes\":{\"base\":60,\"mantissas\":[1,2,5,10,15,20,30],\"max_interval\":1800000.0,\"min_interval\":1000.0,\"num_minor_ticks\":0},\"id\":\"1102\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"end\":1639627167099.7043,\"start\":1639607570390.3516},\"id\":\"1008\",\"type\":\"Range1d\"},{\"attributes\":{\"axis\":{\"id\":\"1016\"},\"coordinates\":null,\"group\":null,\"minor_grid_line_alpha\":0.3,\"minor_grid_line_color\":\"navy\",\"ticker\":null},\"id\":\"1019\",\"type\":\"Grid\"},{\"attributes\":{},\"id\":\"1014\",\"type\":\"LinearScale\"},{\"attributes\":{\"overlay\":{\"id\":\"1063\"},\"x_range\":{\"id\":\"1008\"},\"y_range\":null},\"id\":\"1062\",\"type\":\"RangeTool\"},{\"attributes\":{\"days\":[1,8,15,22]},\"id\":\"1106\",\"type\":\"DaysTicker\"},{\"attributes\":{\"bottom_units\":\"screen\",\"coordinates\":null,\"fill_alpha\":0.5,\"fill_color\":\"lightgrey\",\"group\":null,\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":1.0,\"line_color\":\"black\",\"line_dash\":[4,4],\"line_width\":2,\"right_units\":\"screen\",\"syncable\":false,\"top_units\":\"screen\"},\"id\":\"1029\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"base\":24,\"mantissas\":[1,2,4,6,8,12],\"max_interval\":43200000.0,\"min_interval\":3600000.0,\"num_minor_ticks\":0},\"id\":\"1090\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"fill_alpha\":{\"value\":0.2},\"fill_color\":{\"value\":\"navy\"},\"hatch_alpha\":{\"value\":0.2},\"hatch_color\":{\"value\":\"navy\"},\"line_alpha\":{\"value\":0.2},\"line_color\":{\"value\":\"navy\"},\"marker\":{\"value\":\"diamond\"},\"size\":{\"value\":10},\"x\":{\"field\":\"Timestamp\"},\"y\":{\"field\":\"y_index\"}},\"id\":\"1072\",\"type\":\"Scatter\"},{\"attributes\":{\"coordinates\":null,\"fill_alpha\":0.2,\"fill_color\":\"navy\",\"group\":null,\"level\":\"overlay\",\"line_alpha\":1.0,\"line_color\":\"black\",\"line_dash\":[2,2],\"line_width\":0.5,\"syncable\":false},\"id\":\"1063\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"months\":[0,2,4,6,8,10]},\"id\":\"1109\",\"type\":\"MonthsTicker\"},{\"attributes\":{\"base\":60,\"mantissas\":[1,2,5,10,15,20,30],\"max_interval\":1800000.0,\"min_interval\":1000.0,\"num_minor_ticks\":0},\"id\":\"1089\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"mantissas\":[1,2,5],\"max_interval\":500.0,\"num_minor_ticks\":0},\"id\":\"1088\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"overlay\":{\"id\":\"1029\"}},\"id\":\"1025\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"axis_label\":\"Event Time\",\"coordinates\":null,\"formatter\":{\"id\":\"1067\"},\"group\":null,\"major_label_policy\":{\"id\":\"1082\"},\"ticker\":{\"id\":\"1017\"}},\"id\":\"1016\",\"type\":\"DatetimeAxis\"},{\"attributes\":{\"tools\":[{\"id\":\"1004\"},{\"id\":\"1024\"},{\"id\":\"1025\"},{\"id\":\"1026\"},{\"id\":\"1027\"},{\"id\":\"1028\"}]},\"id\":\"1030\",\"type\":\"Toolbar\"},{\"attributes\":{\"below\":[{\"id\":\"1048\"},{\"id\":\"1053\"}],\"center\":[{\"id\":\"1051\"}],\"height\":120,\"renderers\":[{\"id\":\"1060\"}],\"title\":{\"id\":\"1038\"},\"toolbar\":{\"id\":\"1052\"},\"toolbar_location\":null,\"width\":900,\"x_range\":{\"id\":\"1040\"},\"x_scale\":{\"id\":\"1044\"},\"y_range\":{\"id\":\"1042\"},\"y_scale\":{\"id\":\"1046\"}},\"id\":\"1037\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"dimensions\":\"width\"},\"id\":\"1028\",\"type\":\"PanTool\"},{\"attributes\":{\"coordinates\":null,\"group\":null,\"text\":\"Event Timeline\"},\"id\":\"1006\",\"type\":\"Title\"},{\"attributes\":{\"callback\":null,\"formatters\":{\"@Timestamp\":\"datetime\"},\"tooltips\":[[\"Timestamp\",\"@Timestamp{%F %T.%3N}\"]]},\"id\":\"1004\",\"type\":\"HoverTool\"},{\"attributes\":{},\"id\":\"1012\",\"type\":\"LinearScale\"}],\"root_ids\":[\"1075\"]},\"title\":\"Bokeh Application\",\"version\":\"2.4.2\"}};\n const render_items = [{\"docid\":\"d6e21d0a-d676-473e-948e-daea2d8fbf3f\",\"root_ids\":[\"1075\"],\"roots\":{\"1075\":\"359feaf4-3ee5-4c0b-9a03-5a15d59d26f0\"}}];\n root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n\n }\n if (root.Bokeh !== undefined) {\n embed_document(root);\n } else {\n let attempts = 0;\n const timer = setInterval(function(root) {\n if (root.Bokeh !== undefined) {\n clearInterval(timer);\n embed_document(root);\n } else {\n attempts++;\n if (attempts > 100) {\n clearInterval(timer);\n console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\");\n }\n }\n }, 10, root)\n }\n})(window);", - "application/vnd.bokehjs_exec.v0+json": "" - }, - "metadata": { - "application/vnd.bokehjs_exec.v0+json": { - "id": "1075" - } - }, - "output_type": "display_data" - } - ], - "source": [ - "from msticpy.vis.timeline import display_timeline\n", - "\n", - "mde_df = pd.read_pickle(\"e:/src/msticpy/tests/testdata/mde_proc_pub.pkl\")\n", - "\n", - "plot = display_timeline(data=mde_df, time_column=\"Timestamp\")" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
Figure(
id = '1005', …)
above = [],
align = 'start',
aspect_ratio = None,
aspect_scale = 1,
background = None,
background_fill_alpha = 1.0,
background_fill_color = '#ffffff',
below = [DatetimeAxis(id='1016', ...)],
border_fill_alpha = 1.0,
border_fill_color = '#ffffff',
center = [Grid(id='1019', ...), Grid(id='1023', ...)],
css_classes = [],
disabled = False,
extra_x_ranges = {},
extra_x_scales = {},
extra_y_ranges = {},
extra_y_scales = {},
frame_height = None,
frame_width = None,
height = 300,
height_policy = 'auto',
hidpi = True,
inner_height = 0,
inner_width = 0,
js_event_callbacks = {},
js_property_callbacks = {},
left = [LinearAxis(id='1020', ...)],
lod_factor = 10,
lod_interval = 300,
lod_threshold = 2000,
lod_timeout = 500,
margin = (0, 0, 0, 0),
match_aspect = False,
max_height = None,
max_width = None,
min_border = 5,
min_border_bottom = None,
min_border_left = 50,
min_border_right = None,
min_border_top = None,
min_height = None,
min_width = None,
name = None,
outer_height = 0,
outer_width = 0,
outline_line_alpha = 1.0,
outline_line_cap = 'butt',
outline_line_color = '#e5e5e5',
outline_line_dash = [],
outline_line_dash_offset = 0,
outline_line_join = 'bevel',
outline_line_width = 1,
output_backend = 'canvas',
renderers = [GlyphRenderer(id='1073', ...)],
reset_policy = 'standard',
right = [],
sizing_mode = None,
subscribed_events = [],
syncable = True,
tags = [],
title = Title(id='1006', ...),
title_location = 'above',
toolbar = Toolbar(id='1030', ...),
toolbar_location = 'right',
toolbar_sticky = True,
visible = True,
width = 900,
width_policy = 'auto',
x_range = Range1d(id='1008', ...),
x_scale = LinearScale(id='1012', ...),
y_range = Range1d(id='1010', ...),
y_scale = LinearScale(id='1014', ...))
\n", - "\n" - ], - "text/plain": [ - "Figure(id='1005', ...)" - ] - }, - "execution_count": 42, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "[attr for attr in dir(plot) if not attr.startswith(\"_\")]\n", - "fig = plot.children[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "7bdcf5ad89444b7c89e9883e24ce175c", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "IntSlider(value=0, continuous_update=False, description='V Offset:', max=10, min=-10, orientation='vertical')" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "widgets.IntSlider(\n", - " value=0,\n", - " min=-10,\n", - " max=10,\n", - " step=1,\n", - " description='H Offset:',\n", - " disabled=False,\n", - " continuous_update=False,\n", - " orientation='horizontal',\n", - " readout=True,\n", - " readout_format='d'\n", - ")\n", - "widgets.IntSlider(\n", - " value=0,\n", - " min=-10,\n", - " max=10,\n", - " step=1,\n", - " description='V Offset:',\n", - " disabled=False,\n", - " continuous_update=False,\n", - " orientation='vertical',\n", - " readout=True,\n", - " readout_format='d'\n", - ")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "msticpy", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.13 (default, Mar 28 2022, 06:59:08) [MSC v.1916 64 bit (AMD64)]" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "4b6ddaa4f4daa4a6c6674a452b84f1f4f3e0236ad39c89274d27c46b8457b97f" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 73dfea5541916f7d17657c3763dcb766dfd9d8dc Mon Sep 17 00:00:00 2001 From: ianhelle Date: Fri, 22 Nov 2024 12:46:12 +0100 Subject: [PATCH 8/8] Reverting to classic Dict type for cast call --- msticpy/context/azure/sentinel_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/msticpy/context/azure/sentinel_utils.py b/msticpy/context/azure/sentinel_utils.py index 4ffc018ec..179b659c2 100644 --- a/msticpy/context/azure/sentinel_utils.py +++ b/msticpy/context/azure/sentinel_utils.py @@ -15,7 +15,7 @@ import pandas as pd from azure.common.exceptions import CloudError from azure.mgmt.core import tools as az_tools -from typing_extensions import Self, cast +from typing_extensions import Dict, Self, cast from ..._version import VERSION from ...auth.azure_auth_core import AzureCloudConfig @@ -339,8 +339,8 @@ def parse_resource_id(res_id: str) -> dict[str, Any]: """Extract components from workspace resource ID.""" if not res_id.startswith("/"): res_id = f"/{res_id}" - res_id_parts: dict[str, str] = cast( - dict[str, str], az_tools.parse_resource_id(res_id) + res_id_parts: Dict[str, str] = cast( + Dict[str, str], az_tools.parse_resource_id(res_id) ) workspace_name: str | None = None if (