diff --git a/bigframes/pandas/__init__.py b/bigframes/pandas/__init__.py index 9da2204a71..8aab50b874 100644 --- a/bigframes/pandas/__init__.py +++ b/bigframes/pandas/__init__.py @@ -362,6 +362,13 @@ def reset_session(): reset_session.__doc__ = global_session.close_session.__doc__ +def job_history() -> pandas.DataFrame: + return global_session.with_default_session(bigframes.session.Session.job_history) + + +job_history.__doc__ = inspect.getdoc(bigframes.session.Session.job_history) + + # SQL Compilation uses recursive algorithms on deep trees # 10M tree depth should be sufficient to generate any sql that is under bigquery limit # Note: This limit does not have the desired effect on Python 3.12 in @@ -385,6 +392,7 @@ def reset_session(): deploy_remote_function, deploy_udf, get_default_session_id, + job_history, get_dummies, merge, qcut, @@ -419,6 +427,7 @@ def reset_session(): "deploy_remote_function", "deploy_udf", "get_default_session_id", + "job_history", "get_dummies", "merge", "qcut", diff --git a/bigframes/session/__init__.py b/bigframes/session/__init__.py index 757bb50a94..7466527aed 100644 --- a/bigframes/session/__init__.py +++ b/bigframes/session/__init__.py @@ -366,6 +366,10 @@ def slot_millis_sum(self): """The sum of all slot time used by bigquery jobs in this session.""" return self._metrics.slot_millis + def job_history(self) -> pandas.DataFrame: + """Returns a list of BigQuery jobs initiated by BigFrames in the current session.""" + return pandas.DataFrame([job.__dict__ for job in self._metrics.jobs]) + @property def _allows_ambiguity(self) -> bool: return self._allow_ambiguity diff --git a/bigframes/session/_io/bigquery/read_gbq_table.py b/bigframes/session/_io/bigquery/read_gbq_table.py index e12fe502c0..2ca48a5d3b 100644 --- a/bigframes/session/_io/bigquery/read_gbq_table.py +++ b/bigframes/session/_io/bigquery/read_gbq_table.py @@ -326,6 +326,7 @@ def check_if_index_columns_are_unique( index_cols: List[str], *, publisher: bigframes.core.events.Publisher, + metrics: Optional[bigframes.session.metrics.ExecutionMetrics] = None, ) -> Tuple[str, ...]: import bigframes.core.sql import bigframes.session._io.bigquery @@ -341,7 +342,7 @@ def check_if_index_columns_are_unique( timeout=None, location=None, project=None, - metrics=None, + metrics=metrics, query_with_job=False, publisher=publisher, ) diff --git a/bigframes/session/loader.py b/bigframes/session/loader.py index 9c18d727c8..01ae12dd55 100644 --- a/bigframes/session/loader.py +++ b/bigframes/session/loader.py @@ -562,6 +562,9 @@ def _start_generic_job(self, job: formatting_helpers.GenericJob): else: job.result() + if self._metrics is not None and isinstance(job, google.cloud.bigquery.job.Job): + self._metrics.count_job_stats(query_job=job) + @overload def read_gbq_table( # type: ignore[overload-overlap] self, @@ -878,6 +881,7 @@ def read_gbq_table( table=table, index_cols=index_cols, publisher=self._publisher, + metrics=self._metrics, ) if publish_execution: self._publisher.publish( diff --git a/bigframes/session/metrics.py b/bigframes/session/metrics.py index 8d43a83d73..e03ee53cce 100644 --- a/bigframes/session/metrics.py +++ b/bigframes/session/metrics.py @@ -15,16 +15,42 @@ from __future__ import annotations import dataclasses +import datetime import os -from typing import Optional, Tuple +from typing import Any, Mapping, Optional, Tuple import google.cloud.bigquery as bigquery -import google.cloud.bigquery.job as bq_job import google.cloud.bigquery.table as bq_table LOGGING_NAME_ENV_VAR = "BIGFRAMES_PERFORMANCE_LOG_NAME" +@dataclasses.dataclass +class JobMetadata: + job_id: Optional[str] = None + query_id: Optional[str] = None + location: Optional[str] = None + project: Optional[str] = None + creation_time: Optional[datetime.datetime] = None + start_time: Optional[datetime.datetime] = None + end_time: Optional[datetime.datetime] = None + duration_seconds: Optional[float] = None + status: Optional[str] = None + total_bytes_processed: Optional[int] = None + total_slot_ms: Optional[int] = None + job_type: Optional[str] = None + error_result: Optional[Mapping[str, Any]] = None + cached: Optional[bool] = None + job_url: Optional[str] = None + query: Optional[str] = None + destination_table: Optional[str] = None + source_uris: Optional[list[str]] = None + input_files: Optional[int] = None + input_bytes: Optional[int] = None + output_rows: Optional[int] = None + source_format: Optional[str] = None + + @dataclasses.dataclass class ExecutionMetrics: execution_count: int = 0 @@ -32,10 +58,11 @@ class ExecutionMetrics: bytes_processed: int = 0 execution_secs: float = 0 query_char_count: int = 0 + jobs: list[JobMetadata] = dataclasses.field(default_factory=list) def count_job_stats( self, - query_job: Optional[bq_job.QueryJob] = None, + query_job: Optional[bigquery.job.Job] = None, row_iterator: Optional[bq_table.RowIterator] = None, ): if query_job is None: @@ -57,21 +84,130 @@ def count_job_stats( self.slot_millis += slot_millis self.execution_secs += exec_seconds + self.jobs.append( + JobMetadata( + job_id=getattr(row_iterator, "job_id", None), + query_id=getattr(row_iterator, "query_id", None), + location=getattr(row_iterator, "location", None), + project=getattr(row_iterator, "project", None), + creation_time=created, + start_time=getattr(row_iterator, "started", None), + end_time=ended, + duration_seconds=exec_seconds, + status="DONE", + total_bytes_processed=bytes_processed, + total_slot_ms=slot_millis, + job_type="query", + cached=getattr(row_iterator, "cache_hit", None), + query=getattr(row_iterator, "query", None), + job_url=f"https://console.cloud.google.com/bigquery?project={getattr(row_iterator, 'project', '')}&j=bq:{getattr(row_iterator, 'location', '')}:{getattr(row_iterator, 'job_id', '')}&page=queryresults" + if getattr(row_iterator, "job_id", None) + else None, + ) + ) + elif query_job.configuration.dry_run: - query_char_count = len(query_job.query) + query_char_count = len(getattr(query_job, "query", "")) # TODO(tswast): Pass None after making benchmark publishing robust to missing data. bytes_processed = 0 slot_millis = 0 exec_seconds = 0.0 - elif (stats := get_performance_stats(query_job)) is not None: - query_char_count, bytes_processed, slot_millis, exec_seconds = stats + elif isinstance(query_job, bigquery.QueryJob): + if (stats := get_performance_stats(query_job)) is not None: + query_char_count, bytes_processed, slot_millis, exec_seconds = stats + self.execution_count += 1 + self.query_char_count += query_char_count or 0 + self.bytes_processed += bytes_processed or 0 + self.slot_millis += slot_millis or 0 + self.execution_secs += exec_seconds or 0 + + self.jobs.append( + JobMetadata( + job_id=query_job.job_id, + location=query_job.location, + project=query_job.project, + creation_time=query_job.created, + start_time=query_job.started, + end_time=query_job.ended, + duration_seconds=exec_seconds, + status=query_job.state, + total_bytes_processed=bytes_processed, + total_slot_ms=slot_millis, + job_type=query_job.job_type, + error_result=query_job.error_result, + cached=query_job.cache_hit, + query=query_job.query, + destination_table=str(query_job.destination) + if query_job.destination + else None, + job_url=f"https://console.cloud.google.com/bigquery?project={query_job.project}&j=bq:{query_job.location}:{query_job.job_id}&page=queryresults", + ) + ) + + else: + # Handle other job types (e.g. LoadJob) self.execution_count += 1 - self.query_char_count += query_char_count or 0 - self.bytes_processed += bytes_processed or 0 - self.slot_millis += slot_millis or 0 - self.execution_secs += exec_seconds or 0 + duration = ( + (query_job.ended - query_job.created).total_seconds() + if query_job.ended and query_job.created + else None + ) + + job_metadata = JobMetadata( + job_id=query_job.job_id, + location=query_job.location, + project=query_job.project, + creation_time=query_job.created, + start_time=query_job.started, + end_time=query_job.ended, + duration_seconds=duration, + status=query_job.state, + job_type=query_job.job_type, + error_result=query_job.error_result, + job_url=f"https://console.cloud.google.com/bigquery?project={query_job.project}&j=bq:{query_job.location}:{query_job.job_id}&page=queryresults", + ) + + if isinstance(query_job, bigquery.LoadJob): + job_metadata.output_rows = getattr(query_job, "output_rows", None) + job_metadata.input_files = getattr(query_job, "input_files", None) + job_metadata.input_bytes = getattr(query_job, "input_bytes", None) + job_metadata.destination_table = ( + str(query_job.destination) if query_job.destination else None + ) + if query_job.source_uris: + job_metadata.source_uris = list(query_job.source_uris) + if query_job.configuration and hasattr( + query_job.configuration, "source_format" + ): + job_metadata.source_format = query_job.configuration.source_format + + self.jobs.append(job_metadata) + + # For pytest runs only, log information about the query job + # to a file in order to create a performance report. + if ( + isinstance(query_job, bigquery.QueryJob) + and not query_job.configuration.dry_run + ): + stats = get_performance_stats(query_job) + if stats: + write_stats_to_disk( + query_char_count=stats[0], + bytes_processed=stats[1], + slot_millis=stats[2], + exec_seconds=stats[3], + ) + elif row_iterator is not None: + bytes_processed = getattr(row_iterator, "total_bytes_processed", 0) or 0 + query_char_count = len(getattr(row_iterator, "query", "") or "") + slot_millis = getattr(row_iterator, "slot_millis", 0) or 0 + created = getattr(row_iterator, "created", None) + ended = getattr(row_iterator, "ended", None) + exec_seconds = ( + (ended - created).total_seconds() if created and ended else 0.0 + ) write_stats_to_disk( query_char_count=query_char_count, bytes_processed=bytes_processed, @@ -79,20 +215,6 @@ def count_job_stats( exec_seconds=exec_seconds, ) - else: - # TODO(tswast): Pass None after making benchmark publishing robust to missing data. - bytes_processed = 0 - query_char_count = 0 - slot_millis = 0 - exec_seconds = 0 - - write_stats_to_disk( - query_char_count=query_char_count, - bytes_processed=bytes_processed, - slot_millis=slot_millis, - exec_seconds=exec_seconds, - ) - def get_performance_stats( query_job: bigquery.QueryJob, diff --git a/notebooks/dataframes/job_history.ipynb b/notebooks/dataframes/job_history.ipynb new file mode 100644 index 0000000000..77c1f71d91 --- /dev/null +++ b/notebooks/dataframes/job_history.ipynb @@ -0,0 +1,1349 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Job History Manual Test\\n\n", + "\\n\n", + "This notebook demonstrates and manually tests the `bigframes.pandas.job_history()` functionality." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import bigframes.pandas as bpd\n", + "\n", + "# Set options if needed, e.g. project/location\n", + "# bpd.options.bigquery.project = \"YOUR_PROJECT\"\n", + "# bpd.options.bigquery.location = \"US\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Trigger a Query Job (read_gbq)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 0 Bytes in a moment of slot time.\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 0 Bytes in 9 seconds of slot time. [Job bigframes-dev:US.e05becac-dd84-42ab-ac63-5c14cac5d1ec details]\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 32 Bytes in a moment of slot time.\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Starting." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ab
012
\n", + "

1 rows × 2 columns

\n", + "
[1 rows x 2 columns in total]" + ], + "text/plain": [ + " a b\n", + "0 1 2\n", + "\n", + "[1 rows x 2 columns]" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = bpd.read_gbq(\"SELECT 1 as a, 2 as b\")\n", + "df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Trigger a Load Job (read_pandas)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 0 Bytes in 10 seconds of slot time. [Job bigframes-dev:US.fb392a60-d669-4a24-acec-ec9206c313dc details]\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 81 Bytes in a moment of slot time.\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Starting." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
col1col2
01a
12b
23c
\n", + "

3 rows × 2 columns

\n", + "
[3 rows x 2 columns in total]" + ], + "text/plain": [ + " col1 col2\n", + "0 1 a\n", + "1 2 b\n", + "2 3 c\n", + "\n", + "[3 rows x 2 columns]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "local_df = pd.DataFrame({'col1': [1, 2, 3], 'col2': ['a', 'b', 'c']})\n", + "bf_df = bpd.read_pandas(local_df)\n", + "bf_df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Trigger a Computation (Computation Job)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 0 Bytes in 8 seconds of slot time. [Job bigframes-dev:US.2a0ecc0a-02b2-46e6-a50b-710f2168715c details]\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "✅ Completed. \n", + " Query processed 57 Bytes in a moment of slot time.\n", + " " + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Starting." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
col1
col2
a1
b2
c3
\n", + "

3 rows × 1 columns

\n", + "
[3 rows x 1 columns in total]" + ], + "text/plain": [ + " col1\n", + "col2 \n", + "a 1\n", + "b 2\n", + "c 3\n", + "\n", + "[3 rows x 1 columns]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Perform a simple aggregation to trigger a computation\n", + "agg_df = bf_df.groupby('col2').sum()\n", + "agg_df.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Check Job History" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "job_id", + "rawType": "object", + "type": "unknown" + }, + { + "name": "query_id", + "rawType": "object", + "type": "unknown" + }, + { + "name": "location", + "rawType": "object", + "type": "string" + }, + { + "name": "project", + "rawType": "object", + "type": "unknown" + }, + { + "name": "creation_time", + "rawType": "datetime64[ns, UTC]", + "type": "unknown" + }, + { + "name": "start_time", + "rawType": "datetime64[ns, UTC]", + "type": "unknown" + }, + { + "name": "end_time", + "rawType": "datetime64[ns, UTC]", + "type": "unknown" + }, + { + "name": "duration_seconds", + "rawType": "float64", + "type": "float" + }, + { + "name": "status", + "rawType": "object", + "type": "string" + }, + { + "name": "total_bytes_processed", + "rawType": "int64", + "type": "integer" + }, + { + "name": "total_slot_ms", + "rawType": "int64", + "type": "integer" + }, + { + "name": "job_type", + "rawType": "object", + "type": "string" + }, + { + "name": "error_result", + "rawType": "object", + "type": "unknown" + }, + { + "name": "cached", + "rawType": "object", + "type": "unknown" + }, + { + "name": "job_url", + "rawType": "object", + "type": "unknown" + }, + { + "name": "query", + "rawType": "object", + "type": "string" + }, + { + "name": "destination_table", + "rawType": "object", + "type": "unknown" + }, + { + "name": "source_uris", + "rawType": "object", + "type": "unknown" + }, + { + "name": "input_files", + "rawType": "object", + "type": "unknown" + }, + { + "name": "input_bytes", + "rawType": "object", + "type": "unknown" + }, + { + "name": "output_rows", + "rawType": "object", + "type": "unknown" + }, + { + "name": "source_format", + "rawType": "object", + "type": "unknown" + } + ], + "ref": "c144a4f5-fd05-45f0-87e0-2d203c6e013c", + "rows": [ + [ + "0", + null, + "DYKpEU608_foQAqHltOXN381FFz4!19c2c4c8e93", + "US", + null, + "2026-02-05 05:35:40.650000+00:00", + "2026-02-05 05:35:40.691000+00:00", + "2026-02-05 05:35:40.782000+00:00", + "0.132", + "DONE", + "0", + "9", + "query", + null, + null, + null, + "SELECT 1 as a, 2 as b", + null, + null, + null, + null, + null, + null + ], + [ + "1", + "e05becac-dd84-42ab-ac63-5c14cac5d1ec", + null, + "US", + "bigframes-dev", + "2026-02-05 05:35:42.322000+00:00", + "2026-02-05 05:35:42.577000+00:00", + "2026-02-05 05:35:43.971000+00:00", + "1.649", + "DONE", + "0", + "9030", + "query", + null, + "False", + "https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:US:e05becac-dd84-42ab-ac63-5c14cac5d1ec&page=queryresults", + "SELECT\n`bfuid_col_1` AS `bfuid_col_1`,\n`a` AS `a`,\n`b` AS `b`,\n`bfuid_col_3` AS `bfuid_col_4`,\n`bfuid_col_2` AS `bfuid_col_2`\nFROM\n(SELECT\n `t0`.`a`,\n `t0`.`b`,\n `t0`.`bfuid_col_1`,\n `t0`.`bfuid_col_3`,\n `t0`.`bfuid_col_3` AS `bfuid_col_2`\nFROM (\n SELECT\n *\n FROM UNNEST(ARRAY>[STRUCT(1, 2, 0, 0)]) AS `a`\n) AS `t0`)", + "bigframes-dev._8b037bfb7316dddf9d92b12dcf93e008906bfe52._9f58ff72_dd63_4076_8156_9c560d643d39_bqdf_2052f752-15d5-4923-a96d-76c3132ba6cb", + null, + null, + null, + null, + null + ], + [ + "2", + null, + "nRxqk3Wr_1sYIZlk1kVUbNPmBz-s@19c2c4c9ec4", + "US", + null, + "2026-02-05 05:35:44.690000+00:00", + "2026-02-05 05:35:44.836000+00:00", + "2026-02-05 05:35:45+00:00", + "0.31", + "DONE", + "32", + "0", + "query", + null, + null, + null, + "SELECT\n`bfuid_col_1` AS `bfuid_col_1`,\n`a` AS `a`,\n`b` AS `b`\nFROM\n(SELECT\n `t0`.`bfuid_col_1`,\n `t0`.`a`,\n `t0`.`b`,\n `t0`.`bfuid_col_2` AS `bfuid_col_5`\nFROM `bigframes-dev._8b037bfb7316dddf9d92b12dcf93e008906bfe52._9f58ff72_dd63_4076_8156_9c560d643d39_bqdf_2052f752-15d5-4923-a96d-76c3132ba6cb` AS `t0`)\nORDER BY `bfuid_col_5` ASC NULLS LAST\nLIMIT 10", + null, + null, + null, + null, + null, + null + ], + [ + "3", + "fb392a60-d669-4a24-acec-ec9206c313dc", + null, + "US", + "bigframes-dev", + "2026-02-05 05:35:45.860000+00:00", + "2026-02-05 05:35:46.035000+00:00", + "2026-02-05 05:35:47.340000+00:00", + "1.48", + "DONE", + "0", + "10106", + "query", + null, + "False", + "https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:US:fb392a60-d669-4a24-acec-ec9206c313dc&page=queryresults", + "SELECT\n `t0`.`level_0`,\n `t0`.`column_0`,\n `t0`.`column_1`,\n `t0`.`bfuid_col_8`,\n `t0`.`bfuid_col_8` AS `bfuid_col_7`\nFROM (\n SELECT\n *\n FROM UNNEST(ARRAY>[STRUCT(0, 1, 'a', 0), STRUCT(1, 2, 'b', 1), STRUCT(2, 3, 'c', 2)]) AS `level_0`\n) AS `t0`", + "bigframes-dev._8b037bfb7316dddf9d92b12dcf93e008906bfe52._9f58ff72_dd63_4076_8156_9c560d643d39_bqdf_489567cd-be4c-4803-97ca-73a4e8bfe932", + null, + null, + null, + null, + null + ], + [ + "4", + null, + "XQGOd9iV2zzzatbrBKqFvjAHkO7v^19c2c4cab05", + "US", + null, + "2026-02-05 05:35:47.857000+00:00", + "2026-02-05 05:35:47.973000+00:00", + "2026-02-05 05:35:48.100000+00:00", + "0.243", + "DONE", + "81", + "0", + "query", + null, + null, + null, + "SELECT\n`level_0` AS `level_0`,\n`column_0` AS `column_0`,\n`column_1` AS `column_1`\nFROM\n(SELECT\n `t0`.`level_0`,\n `t0`.`column_0`,\n `t0`.`column_1`,\n `t0`.`bfuid_col_7` AS `bfuid_col_9`\nFROM `bigframes-dev._8b037bfb7316dddf9d92b12dcf93e008906bfe52._9f58ff72_dd63_4076_8156_9c560d643d39_bqdf_489567cd-be4c-4803-97ca-73a4e8bfe932` AS `t0`)\nORDER BY `bfuid_col_9` ASC NULLS LAST\nLIMIT 10", + null, + null, + null, + null, + null, + null + ], + [ + "5", + "2a0ecc0a-02b2-46e6-a50b-710f2168715c", + null, + "US", + "bigframes-dev", + "2026-02-05 05:35:49.085000+00:00", + "2026-02-05 05:35:49.336000+00:00", + "2026-02-05 05:35:50.699000+00:00", + "1.614", + "DONE", + "0", + "8983", + "query", + null, + "False", + "https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:US:2a0ecc0a-02b2-46e6-a50b-710f2168715c&page=queryresults", + "SELECT\n`column_1` AS `column_1`,\n`bfuid_col_11` AS `bfuid_col_11`,\n`bfuid_col_13` AS `bfuid_col_14`,\n`bfuid_col_12` AS `bfuid_col_12`\nFROM\n(SELECT\n `t5`.`column_1`,\n `t5`.`bfuid_col_11`,\n `t5`.`bfuid_col_13`,\n ROW_NUMBER() OVER (ORDER BY `t5`.`bfuid_col_13` IS NULL ASC, `t5`.`bfuid_col_13` ASC) - 1 AS `bfuid_col_12`\nFROM (\n SELECT\n *\n FROM (\n SELECT\n `t3`.`column_1`,\n `t3`.`bfuid_col_11`,\n ROW_NUMBER() OVER (ORDER BY `t3`.`column_1` IS NULL ASC, `t3`.`column_1` ASC) - 1 AS `bfuid_col_13`\n FROM (\n SELECT\n *\n FROM (\n SELECT\n `t1`.`column_1`,\n COALESCE(SUM(`t1`.`column_0`), 0) AS `bfuid_col_11`\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM UNNEST(ARRAY>[STRUCT(1, 'a'), STRUCT(2, 'b'), STRUCT(3, 'c')]) AS `column_0`\n ) AS `t0`\n ) AS `t1`\n GROUP BY\n 1\n ) AS `t2`\n WHERE\n (\n `t2`.`column_1`\n ) IS NOT NULL\n ) AS `t3`\n ) AS `t4`\n WHERE\n `t4`.`bfuid_col_13` < 5\n) AS `t5`)", + "bigframes-dev._8b037bfb7316dddf9d92b12dcf93e008906bfe52._9f58ff72_dd63_4076_8156_9c560d643d39_bqdf_58201d36-f0be-4891-a011-d0bdbd95b5d9", + null, + null, + null, + null, + null + ], + [ + "6", + null, + "6ogNo1ZJgaNHIqm0YlUi3lpQGO20%19c2c4cb801", + "US", + null, + "2026-02-05 05:35:51.185000+00:00", + "2026-02-05 05:35:51.297000+00:00", + "2026-02-05 05:35:51.446000+00:00", + "0.261", + "DONE", + "57", + "0", + "query", + null, + null, + null, + "SELECT\n`column_1` AS `column_1`,\n`bfuid_col_11` AS `bfuid_col_11`\nFROM\n(SELECT\n `t0`.`column_1`,\n `t0`.`bfuid_col_11`,\n `t0`.`bfuid_col_12` AS `bfuid_col_15`\nFROM `bigframes-dev._8b037bfb7316dddf9d92b12dcf93e008906bfe52._9f58ff72_dd63_4076_8156_9c560d643d39_bqdf_58201d36-f0be-4891-a011-d0bdbd95b5d9` AS `t0`)\nORDER BY `bfuid_col_15` ASC NULLS LAST\nLIMIT 10", + null, + null, + null, + null, + null, + null + ] + ], + "shape": { + "columns": 22, + "rows": 7 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
job_idquery_idlocationprojectcreation_timestart_timeend_timeduration_secondsstatustotal_bytes_processed...error_resultcachedjob_urlquerydestination_tablesource_urisinput_filesinput_bytesoutput_rowssource_format
0NoneDYKpEU608_foQAqHltOXN381FFz4!19c2c4c8e93USNone2026-02-05 05:35:40.650000+00:002026-02-05 05:35:40.691000+00:002026-02-05 05:35:40.782000+00:000.132DONE0...NoneNoneNoneSELECT 1 as a, 2 as bNoneNoneNoneNoneNoneNone
1e05becac-dd84-42ab-ac63-5c14cac5d1ecNoneUSbigframes-dev2026-02-05 05:35:42.322000+00:002026-02-05 05:35:42.577000+00:002026-02-05 05:35:43.971000+00:001.649DONE0...NoneFalsehttps://console.cloud.google.com/bigquery?proj...SELECT\\n`bfuid_col_1` AS `bfuid_col_1`,\\n`a` A...bigframes-dev._8b037bfb7316dddf9d92b12dcf93e00...NoneNoneNoneNoneNone
2NonenRxqk3Wr_1sYIZlk1kVUbNPmBz-s@19c2c4c9ec4USNone2026-02-05 05:35:44.690000+00:002026-02-05 05:35:44.836000+00:002026-02-05 05:35:45+00:000.310DONE32...NoneNoneNoneSELECT\\n`bfuid_col_1` AS `bfuid_col_1`,\\n`a` A...NoneNoneNoneNoneNoneNone
3fb392a60-d669-4a24-acec-ec9206c313dcNoneUSbigframes-dev2026-02-05 05:35:45.860000+00:002026-02-05 05:35:46.035000+00:002026-02-05 05:35:47.340000+00:001.480DONE0...NoneFalsehttps://console.cloud.google.com/bigquery?proj...SELECT\\n `t0`.`level_0`,\\n `t0`.`column_0`,\\...bigframes-dev._8b037bfb7316dddf9d92b12dcf93e00...NoneNoneNoneNoneNone
4NoneXQGOd9iV2zzzatbrBKqFvjAHkO7v^19c2c4cab05USNone2026-02-05 05:35:47.857000+00:002026-02-05 05:35:47.973000+00:002026-02-05 05:35:48.100000+00:000.243DONE81...NoneNoneNoneSELECT\\n`level_0` AS `level_0`,\\n`column_0` AS...NoneNoneNoneNoneNoneNone
52a0ecc0a-02b2-46e6-a50b-710f2168715cNoneUSbigframes-dev2026-02-05 05:35:49.085000+00:002026-02-05 05:35:49.336000+00:002026-02-05 05:35:50.699000+00:001.614DONE0...NoneFalsehttps://console.cloud.google.com/bigquery?proj...SELECT\\n`column_1` AS `column_1`,\\n`bfuid_col_...bigframes-dev._8b037bfb7316dddf9d92b12dcf93e00...NoneNoneNoneNoneNone
6None6ogNo1ZJgaNHIqm0YlUi3lpQGO20%19c2c4cb801USNone2026-02-05 05:35:51.185000+00:002026-02-05 05:35:51.297000+00:002026-02-05 05:35:51.446000+00:000.261DONE57...NoneNoneNoneSELECT\\n`column_1` AS `column_1`,\\n`bfuid_col_...NoneNoneNoneNoneNoneNone
\n", + "

7 rows × 22 columns

\n", + "
" + ], + "text/plain": [ + " job_id \\\n", + "0 None \n", + "1 e05becac-dd84-42ab-ac63-5c14cac5d1ec \n", + "2 None \n", + "3 fb392a60-d669-4a24-acec-ec9206c313dc \n", + "4 None \n", + "5 2a0ecc0a-02b2-46e6-a50b-710f2168715c \n", + "6 None \n", + "\n", + " query_id location project \\\n", + "0 DYKpEU608_foQAqHltOXN381FFz4!19c2c4c8e93 US None \n", + "1 None US bigframes-dev \n", + "2 nRxqk3Wr_1sYIZlk1kVUbNPmBz-s@19c2c4c9ec4 US None \n", + "3 None US bigframes-dev \n", + "4 XQGOd9iV2zzzatbrBKqFvjAHkO7v^19c2c4cab05 US None \n", + "5 None US bigframes-dev \n", + "6 6ogNo1ZJgaNHIqm0YlUi3lpQGO20%19c2c4cb801 US None \n", + "\n", + " creation_time start_time \\\n", + "0 2026-02-05 05:35:40.650000+00:00 2026-02-05 05:35:40.691000+00:00 \n", + "1 2026-02-05 05:35:42.322000+00:00 2026-02-05 05:35:42.577000+00:00 \n", + "2 2026-02-05 05:35:44.690000+00:00 2026-02-05 05:35:44.836000+00:00 \n", + "3 2026-02-05 05:35:45.860000+00:00 2026-02-05 05:35:46.035000+00:00 \n", + "4 2026-02-05 05:35:47.857000+00:00 2026-02-05 05:35:47.973000+00:00 \n", + "5 2026-02-05 05:35:49.085000+00:00 2026-02-05 05:35:49.336000+00:00 \n", + "6 2026-02-05 05:35:51.185000+00:00 2026-02-05 05:35:51.297000+00:00 \n", + "\n", + " end_time duration_seconds status \\\n", + "0 2026-02-05 05:35:40.782000+00:00 0.132 DONE \n", + "1 2026-02-05 05:35:43.971000+00:00 1.649 DONE \n", + "2 2026-02-05 05:35:45+00:00 0.310 DONE \n", + "3 2026-02-05 05:35:47.340000+00:00 1.480 DONE \n", + "4 2026-02-05 05:35:48.100000+00:00 0.243 DONE \n", + "5 2026-02-05 05:35:50.699000+00:00 1.614 DONE \n", + "6 2026-02-05 05:35:51.446000+00:00 0.261 DONE \n", + "\n", + " total_bytes_processed ... error_result cached \\\n", + "0 0 ... None None \n", + "1 0 ... None False \n", + "2 32 ... None None \n", + "3 0 ... None False \n", + "4 81 ... None None \n", + "5 0 ... None False \n", + "6 57 ... None None \n", + "\n", + " job_url \\\n", + "0 None \n", + "1 https://console.cloud.google.com/bigquery?proj... \n", + "2 None \n", + "3 https://console.cloud.google.com/bigquery?proj... \n", + "4 None \n", + "5 https://console.cloud.google.com/bigquery?proj... \n", + "6 None \n", + "\n", + " query \\\n", + "0 SELECT 1 as a, 2 as b \n", + "1 SELECT\\n`bfuid_col_1` AS `bfuid_col_1`,\\n`a` A... \n", + "2 SELECT\\n`bfuid_col_1` AS `bfuid_col_1`,\\n`a` A... \n", + "3 SELECT\\n `t0`.`level_0`,\\n `t0`.`column_0`,\\... \n", + "4 SELECT\\n`level_0` AS `level_0`,\\n`column_0` AS... \n", + "5 SELECT\\n`column_1` AS `column_1`,\\n`bfuid_col_... \n", + "6 SELECT\\n`column_1` AS `column_1`,\\n`bfuid_col_... \n", + "\n", + " destination_table source_uris input_files \\\n", + "0 None None None \n", + "1 bigframes-dev._8b037bfb7316dddf9d92b12dcf93e00... None None \n", + "2 None None None \n", + "3 bigframes-dev._8b037bfb7316dddf9d92b12dcf93e00... None None \n", + "4 None None None \n", + "5 bigframes-dev._8b037bfb7316dddf9d92b12dcf93e00... None None \n", + "6 None None None \n", + "\n", + " input_bytes output_rows source_format \n", + "0 None None None \n", + "1 None None None \n", + "2 None None None \n", + "3 None None None \n", + "4 None None None \n", + "5 None None None \n", + "6 None None None \n", + "\n", + "[7 rows x 22 columns]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "history = bpd.job_history()\n", + "history" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Verify Specific Columns" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.microsoft.datawrangler.viewer.v0+json": { + "columns": [ + { + "name": "index", + "rawType": "int64", + "type": "integer" + }, + { + "name": "job_id", + "rawType": "object", + "type": "unknown" + }, + { + "name": "job_type", + "rawType": "object", + "type": "string" + }, + { + "name": "creation_time", + "rawType": "datetime64[ns, UTC]", + "type": "unknown" + }, + { + "name": "duration_seconds", + "rawType": "float64", + "type": "float" + }, + { + "name": "total_bytes_processed", + "rawType": "int64", + "type": "integer" + }, + { + "name": "query", + "rawType": "object", + "type": "string" + }, + { + "name": "input_files", + "rawType": "object", + "type": "unknown" + }, + { + "name": "destination_table", + "rawType": "object", + "type": "unknown" + } + ], + "ref": "427bf202-125c-4cec-a72b-64795c7ca522", + "rows": [ + [ + "0", + null, + "query", + "2026-02-05 05:35:40.650000+00:00", + "0.132", + "0", + "SELECT 1 as a, 2 as b", + null, + null + ], + [ + "1", + "e05becac-dd84-42ab-ac63-5c14cac5d1ec", + "query", + "2026-02-05 05:35:42.322000+00:00", + "1.649", + "0", + "SELECT\n`bfuid_col_1` AS `bfuid_col_1`,\n`a` AS `a`,\n`b` AS `b`,\n`bfuid_col_3` AS `bfuid_col_4`,\n`bfuid_col_2` AS `bfuid_col_2`\nFROM\n(SELECT\n `t0`.`a`,\n `t0`.`b`,\n `t0`.`bfuid_col_1`,\n `t0`.`bfuid_col_3`,\n `t0`.`bfuid_col_3` AS `bfuid_col_2`\nFROM (\n SELECT\n *\n FROM UNNEST(ARRAY>[STRUCT(1, 2, 0, 0)]) AS `a`\n) AS `t0`)", + null, + "bigframes-dev._8b037bfb7316dddf9d92b12dcf93e008906bfe52._9f58ff72_dd63_4076_8156_9c560d643d39_bqdf_2052f752-15d5-4923-a96d-76c3132ba6cb" + ], + [ + "2", + null, + "query", + "2026-02-05 05:35:44.690000+00:00", + "0.31", + "32", + "SELECT\n`bfuid_col_1` AS `bfuid_col_1`,\n`a` AS `a`,\n`b` AS `b`\nFROM\n(SELECT\n `t0`.`bfuid_col_1`,\n `t0`.`a`,\n `t0`.`b`,\n `t0`.`bfuid_col_2` AS `bfuid_col_5`\nFROM `bigframes-dev._8b037bfb7316dddf9d92b12dcf93e008906bfe52._9f58ff72_dd63_4076_8156_9c560d643d39_bqdf_2052f752-15d5-4923-a96d-76c3132ba6cb` AS `t0`)\nORDER BY `bfuid_col_5` ASC NULLS LAST\nLIMIT 10", + null, + null + ], + [ + "3", + "fb392a60-d669-4a24-acec-ec9206c313dc", + "query", + "2026-02-05 05:35:45.860000+00:00", + "1.48", + "0", + "SELECT\n `t0`.`level_0`,\n `t0`.`column_0`,\n `t0`.`column_1`,\n `t0`.`bfuid_col_8`,\n `t0`.`bfuid_col_8` AS `bfuid_col_7`\nFROM (\n SELECT\n *\n FROM UNNEST(ARRAY>[STRUCT(0, 1, 'a', 0), STRUCT(1, 2, 'b', 1), STRUCT(2, 3, 'c', 2)]) AS `level_0`\n) AS `t0`", + null, + "bigframes-dev._8b037bfb7316dddf9d92b12dcf93e008906bfe52._9f58ff72_dd63_4076_8156_9c560d643d39_bqdf_489567cd-be4c-4803-97ca-73a4e8bfe932" + ], + [ + "4", + null, + "query", + "2026-02-05 05:35:47.857000+00:00", + "0.243", + "81", + "SELECT\n`level_0` AS `level_0`,\n`column_0` AS `column_0`,\n`column_1` AS `column_1`\nFROM\n(SELECT\n `t0`.`level_0`,\n `t0`.`column_0`,\n `t0`.`column_1`,\n `t0`.`bfuid_col_7` AS `bfuid_col_9`\nFROM `bigframes-dev._8b037bfb7316dddf9d92b12dcf93e008906bfe52._9f58ff72_dd63_4076_8156_9c560d643d39_bqdf_489567cd-be4c-4803-97ca-73a4e8bfe932` AS `t0`)\nORDER BY `bfuid_col_9` ASC NULLS LAST\nLIMIT 10", + null, + null + ], + [ + "5", + "2a0ecc0a-02b2-46e6-a50b-710f2168715c", + "query", + "2026-02-05 05:35:49.085000+00:00", + "1.614", + "0", + "SELECT\n`column_1` AS `column_1`,\n`bfuid_col_11` AS `bfuid_col_11`,\n`bfuid_col_13` AS `bfuid_col_14`,\n`bfuid_col_12` AS `bfuid_col_12`\nFROM\n(SELECT\n `t5`.`column_1`,\n `t5`.`bfuid_col_11`,\n `t5`.`bfuid_col_13`,\n ROW_NUMBER() OVER (ORDER BY `t5`.`bfuid_col_13` IS NULL ASC, `t5`.`bfuid_col_13` ASC) - 1 AS `bfuid_col_12`\nFROM (\n SELECT\n *\n FROM (\n SELECT\n `t3`.`column_1`,\n `t3`.`bfuid_col_11`,\n ROW_NUMBER() OVER (ORDER BY `t3`.`column_1` IS NULL ASC, `t3`.`column_1` ASC) - 1 AS `bfuid_col_13`\n FROM (\n SELECT\n *\n FROM (\n SELECT\n `t1`.`column_1`,\n COALESCE(SUM(`t1`.`column_0`), 0) AS `bfuid_col_11`\n FROM (\n SELECT\n *\n FROM (\n SELECT\n *\n FROM UNNEST(ARRAY>[STRUCT(1, 'a'), STRUCT(2, 'b'), STRUCT(3, 'c')]) AS `column_0`\n ) AS `t0`\n ) AS `t1`\n GROUP BY\n 1\n ) AS `t2`\n WHERE\n (\n `t2`.`column_1`\n ) IS NOT NULL\n ) AS `t3`\n ) AS `t4`\n WHERE\n `t4`.`bfuid_col_13` < 5\n) AS `t5`)", + null, + "bigframes-dev._8b037bfb7316dddf9d92b12dcf93e008906bfe52._9f58ff72_dd63_4076_8156_9c560d643d39_bqdf_58201d36-f0be-4891-a011-d0bdbd95b5d9" + ], + [ + "6", + null, + "query", + "2026-02-05 05:35:51.185000+00:00", + "0.261", + "57", + "SELECT\n`column_1` AS `column_1`,\n`bfuid_col_11` AS `bfuid_col_11`\nFROM\n(SELECT\n `t0`.`column_1`,\n `t0`.`bfuid_col_11`,\n `t0`.`bfuid_col_12` AS `bfuid_col_15`\nFROM `bigframes-dev._8b037bfb7316dddf9d92b12dcf93e008906bfe52._9f58ff72_dd63_4076_8156_9c560d643d39_bqdf_58201d36-f0be-4891-a011-d0bdbd95b5d9` AS `t0`)\nORDER BY `bfuid_col_15` ASC NULLS LAST\nLIMIT 10", + null, + null + ] + ], + "shape": { + "columns": 8, + "rows": 7 + } + }, + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
job_idjob_typecreation_timeduration_secondstotal_bytes_processedqueryinput_filesdestination_table
0Nonequery2026-02-05 05:35:40.650000+00:000.1320SELECT 1 as a, 2 as bNoneNone
1e05becac-dd84-42ab-ac63-5c14cac5d1ecquery2026-02-05 05:35:42.322000+00:001.6490SELECT\\n`bfuid_col_1` AS `bfuid_col_1`,\\n`a` A...Nonebigframes-dev._8b037bfb7316dddf9d92b12dcf93e00...
2Nonequery2026-02-05 05:35:44.690000+00:000.31032SELECT\\n`bfuid_col_1` AS `bfuid_col_1`,\\n`a` A...NoneNone
3fb392a60-d669-4a24-acec-ec9206c313dcquery2026-02-05 05:35:45.860000+00:001.4800SELECT\\n `t0`.`level_0`,\\n `t0`.`column_0`,\\...Nonebigframes-dev._8b037bfb7316dddf9d92b12dcf93e00...
4Nonequery2026-02-05 05:35:47.857000+00:000.24381SELECT\\n`level_0` AS `level_0`,\\n`column_0` AS...NoneNone
52a0ecc0a-02b2-46e6-a50b-710f2168715cquery2026-02-05 05:35:49.085000+00:001.6140SELECT\\n`column_1` AS `column_1`,\\n`bfuid_col_...Nonebigframes-dev._8b037bfb7316dddf9d92b12dcf93e00...
6Nonequery2026-02-05 05:35:51.185000+00:000.26157SELECT\\n`column_1` AS `column_1`,\\n`bfuid_col_...NoneNone
\n", + "
" + ], + "text/plain": [ + " job_id job_type \\\n", + "0 None query \n", + "1 e05becac-dd84-42ab-ac63-5c14cac5d1ec query \n", + "2 None query \n", + "3 fb392a60-d669-4a24-acec-ec9206c313dc query \n", + "4 None query \n", + "5 2a0ecc0a-02b2-46e6-a50b-710f2168715c query \n", + "6 None query \n", + "\n", + " creation_time duration_seconds total_bytes_processed \\\n", + "0 2026-02-05 05:35:40.650000+00:00 0.132 0 \n", + "1 2026-02-05 05:35:42.322000+00:00 1.649 0 \n", + "2 2026-02-05 05:35:44.690000+00:00 0.310 32 \n", + "3 2026-02-05 05:35:45.860000+00:00 1.480 0 \n", + "4 2026-02-05 05:35:47.857000+00:00 0.243 81 \n", + "5 2026-02-05 05:35:49.085000+00:00 1.614 0 \n", + "6 2026-02-05 05:35:51.185000+00:00 0.261 57 \n", + "\n", + " query input_files \\\n", + "0 SELECT 1 as a, 2 as b None \n", + "1 SELECT\\n`bfuid_col_1` AS `bfuid_col_1`,\\n`a` A... None \n", + "2 SELECT\\n`bfuid_col_1` AS `bfuid_col_1`,\\n`a` A... None \n", + "3 SELECT\\n `t0`.`level_0`,\\n `t0`.`column_0`,\\... None \n", + "4 SELECT\\n`level_0` AS `level_0`,\\n`column_0` AS... None \n", + "5 SELECT\\n`column_1` AS `column_1`,\\n`bfuid_col_... None \n", + "6 SELECT\\n`column_1` AS `column_1`,\\n`bfuid_col_... None \n", + "\n", + " destination_table \n", + "0 None \n", + "1 bigframes-dev._8b037bfb7316dddf9d92b12dcf93e00... \n", + "2 None \n", + "3 bigframes-dev._8b037bfb7316dddf9d92b12dcf93e00... \n", + "4 None \n", + "5 bigframes-dev._8b037bfb7316dddf9d92b12dcf93e00... \n", + "6 None " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Display key columns to verify data population\n", + "cols_to_check = [\n", + " 'job_id',\n", + " 'job_type',\n", + " 'creation_time',\n", + " 'duration_seconds',\n", + " 'total_bytes_processed',\n", + " 'query',\n", + " 'input_files', # Should be populated for Load Job\n", + " 'destination_table'\n", + "]\n", + "\n", + "# Filter columns that exist in the history DataFrame\n", + "existing_cols = [col for col in cols_to_check if col in history.columns]\n", + "history[existing_cols]" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/tests/unit/session/test_job_history.py b/tests/unit/session/test_job_history.py new file mode 100644 index 0000000000..31d9aed75a --- /dev/null +++ b/tests/unit/session/test_job_history.py @@ -0,0 +1,195 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import unittest.mock + +import google.cloud.bigquery as bigquery +import pandas + +import bigframes.pandas as bpd +import bigframes.session +import bigframes.session.metrics as metrics + +NOW = datetime.datetime.now(datetime.timezone.utc) + + +def test_pandas_job_history(monkeypatch): + query_job = unittest.mock.create_autospec(bigquery.QueryJob, instance=True) + query_job.job_id = "job_pandas" + query_job.location = "US" + query_job.created = NOW + query_job.started = NOW + query_job.ended = NOW + datetime.timedelta(seconds=1) + query_job.state = "DONE" + query_job.total_bytes_processed = 1024 + query_job.slot_millis = 100 + query_job.job_type = "query" + query_job.error_result = None + query_job.cache_hit = False + query_job.query = "SELECT 1" + query_job.configuration.dry_run = False + + # Mock the clients provider to avoid actual BQ client creation + clients_provider = unittest.mock.create_autospec( + bigframes.session.clients.ClientsProvider + ) + bq_client = unittest.mock.create_autospec(bigquery.Client) + bq_client.project = "test-project" + bq_client.default_query_job_config = bigquery.QueryJobConfig() + bq_client.query_and_wait.return_value = iter([[NOW]]) + clients_provider.bqclient = bq_client + + session = bigframes.session.Session(clients_provider=clients_provider) + + # Mock get_global_session to return our session + monkeypatch.setattr( + bigframes.core.global_session, "get_global_session", lambda: session + ) + + session._metrics.count_job_stats(query_job=query_job) + + df = bpd.job_history() + + assert isinstance(df, pandas.DataFrame) + assert len(df) == 1 + assert df.iloc[0]["job_id"] == "job_pandas" + + +def test_session_job_history(): + query_job = unittest.mock.create_autospec(bigquery.QueryJob, instance=True) + query_job.job_id = "job1" + query_job.location = "US" + query_job.created = NOW + query_job.started = NOW + query_job.ended = NOW + datetime.timedelta(seconds=1) + query_job.state = "DONE" + query_job.total_bytes_processed = 1024 + query_job.slot_millis = 100 + query_job.job_type = "query" + query_job.error_result = None + query_job.cache_hit = False + query_job.query = "SELECT 1" + query_job.configuration.dry_run = False + + # Mock the clients provider to avoid actual BQ client creation + clients_provider = unittest.mock.create_autospec( + bigframes.session.clients.ClientsProvider + ) + # We need to mock bqclient specifically as it's accessed during Session init + bq_client = unittest.mock.create_autospec(bigquery.Client) + bq_client.project = "test-project" + bq_client.default_query_job_config = bigquery.QueryJobConfig() + # Mock clock sync query + bq_client.query_and_wait.return_value = iter([[NOW]]) + clients_provider.bqclient = bq_client + + session = bigframes.session.Session(clients_provider=clients_provider) + session._metrics.count_job_stats(query_job=query_job) + + df = session.job_history() + + assert isinstance(df, pandas.DataFrame) + assert len(df) == 1 + assert df.iloc[0]["job_id"] == "job1" + assert "query_id" in df.columns + assert "creation_time" in df.columns + + +def test_job_history_with_query_job(): + query_job = unittest.mock.create_autospec(bigquery.QueryJob, instance=True) + query_job.job_id = "job1" + query_job.location = "US" + query_job.created = NOW + query_job.started = NOW + datetime.timedelta(seconds=1) + query_job.ended = NOW + datetime.timedelta(seconds=3) + query_job.state = "DONE" + query_job.total_bytes_processed = 1024 + query_job.slot_millis = 100 + query_job.job_type = "query" + query_job.error_result = None + query_job.cache_hit = False + query_job.query = "SELECT 1" + query_job.configuration.dry_run = False + + execution_metrics = metrics.ExecutionMetrics() + execution_metrics.count_job_stats(query_job=query_job) + + assert len(execution_metrics.jobs) == 1 + job = execution_metrics.jobs[0] + assert job.job_id == "job1" + assert job.status == "DONE" + assert job.total_bytes_processed == 1024 + assert job.duration_seconds == 3.0 + + +def test_job_history_with_row_iterator(): + row_iterator = unittest.mock.create_autospec( + bigquery.table.RowIterator, instance=True + ) + row_iterator.job_id = "job2" + row_iterator.query_id = "query2" + row_iterator.location = "US" + row_iterator.created = NOW + row_iterator.started = NOW + datetime.timedelta(seconds=1) + row_iterator.ended = NOW + datetime.timedelta(seconds=2) + row_iterator.total_bytes_processed = 512 + row_iterator.slot_millis = 50 + row_iterator.cache_hit = True + row_iterator.query = "SELECT 2" + + execution_metrics = metrics.ExecutionMetrics() + execution_metrics.count_job_stats(row_iterator=row_iterator) + + assert len(execution_metrics.jobs) == 1 + job = execution_metrics.jobs[0] + assert job.job_id == "job2" + assert job.query_id == "query2" + assert job.status == "DONE" + assert job.cached is True + assert job.duration_seconds == 2.0 + + +def test_job_history_with_load_job(): + load_job = unittest.mock.create_autospec(bigquery.LoadJob, instance=True) + load_job.job_id = "job3" + load_job.location = "US" + load_job.created = NOW + load_job.started = NOW + load_job.ended = NOW + datetime.timedelta(seconds=5) + load_job.state = "DONE" + load_job.job_type = "load" + load_job.error_result = None + load_job.configuration.dry_run = False + load_job.output_rows = 100 + load_job.input_files = 1 + load_job.input_bytes = 1024 + load_job.destination = bigquery.TableReference.from_string("project.dataset.table") + load_job.source_uris = ["gs://bucket/file.csv"] + load_job.configuration.source_format = "CSV" + + execution_metrics = metrics.ExecutionMetrics() + execution_metrics.count_job_stats(query_job=load_job) + + assert len(execution_metrics.jobs) == 1 + job = execution_metrics.jobs[0] + assert job.job_id == "job3" + assert job.job_type == "load" + assert job.duration_seconds == 5.0 + assert job.output_rows == 100 + assert job.input_files == 1 + assert job.input_bytes == 1024 + assert job.destination_table == "project.dataset.table" + assert job.source_uris == ["gs://bucket/file.csv"] + assert job.source_format == "CSV"