diff --git a/bigframes/formatting_helpers.py b/bigframes/formatting_helpers.py
index 1e3cdabdaf6..094493818de 100644
--- a/bigframes/formatting_helpers.py
+++ b/bigframes/formatting_helpers.py
@@ -27,8 +27,6 @@
import humanize
if TYPE_CHECKING:
- from IPython import display
-
import bigframes.core.events
GenericJob = Union[
@@ -134,16 +132,14 @@ def repr_query_job_html(query_job: Optional[bigquery.QueryJob]):
return res
-current_display: Optional[display.HTML] = None
current_display_id: Optional[str] = None
-previous_display_html: str = ""
def progress_callback(
event: bigframes.core.events.Event,
):
"""Displays a progress bar while the query is running"""
- global current_display, current_display_id, previous_display_html
+ global current_display_id
try:
import bigframes._config
@@ -162,59 +158,44 @@ def progress_callback(
if progress_bar == "notebook":
import IPython.display as display
- if (
- isinstance(event, bigframes.core.events.ExecutionStarted)
- or current_display is None
- or current_display_id is None
- ):
- previous_display_html = ""
- current_display_id = str(random.random())
- current_display = display.HTML("Starting.")
- display.display(
- current_display,
- display_id=current_display_id,
- )
+ display_html = None
+
+ if isinstance(event, bigframes.core.events.ExecutionStarted):
+ # Start a new context for progress output.
+ current_display_id = None
+
+ elif isinstance(event, bigframes.core.events.BigQuerySentEvent):
+ display_html = render_bqquery_sent_event_html(event)
- if isinstance(event, bigframes.core.events.BigQuerySentEvent):
- previous_display_html = render_bqquery_sent_event_html(event)
- display.update_display(
- display.HTML(previous_display_html),
- display_id=current_display_id,
- )
elif isinstance(event, bigframes.core.events.BigQueryRetryEvent):
- previous_display_html = render_bqquery_retry_event_html(event)
- display.update_display(
- display.HTML(previous_display_html),
- display_id=current_display_id,
- )
+ display_html = render_bqquery_retry_event_html(event)
+
elif isinstance(event, bigframes.core.events.BigQueryReceivedEvent):
- previous_display_html = render_bqquery_received_event_html(event)
- display.update_display(
- display.HTML(previous_display_html),
- display_id=current_display_id,
- )
+ display_html = render_bqquery_received_event_html(event)
+
elif isinstance(event, bigframes.core.events.BigQueryFinishedEvent):
- previous_display_html = render_bqquery_finished_event_html(event)
- display.update_display(
- display.HTML(previous_display_html),
- display_id=current_display_id,
- )
- elif isinstance(event, bigframes.core.events.ExecutionFinished):
- if previous_display_html:
+ display_html = render_bqquery_finished_event_html(event)
+
+ elif isinstance(event, bigframes.core.events.SessionClosed):
+ display_html = f"Session {event.session_id} closed."
+
+ if display_html:
+ if current_display_id:
display.update_display(
- display.HTML(f"✅ Completed. {previous_display_html}"),
+ display.HTML(display_html),
+ display_id=current_display_id,
+ )
+ else:
+ current_display_id = str(random.random())
+ display.display(
+ display.HTML(display_html),
display_id=current_display_id,
)
- elif isinstance(event, bigframes.core.events.SessionClosed):
- display.update_display(
- display.HTML(f"Session {event.session_id} closed."),
- display_id=current_display_id,
- )
elif progress_bar == "terminal":
- if isinstance(event, bigframes.core.events.ExecutionStarted):
- print("Starting execution.")
- elif isinstance(event, bigframes.core.events.BigQuerySentEvent):
+ message = None
+
+ if isinstance(event, bigframes.core.events.BigQuerySentEvent):
message = render_bqquery_sent_event_plaintext(event)
print(message)
elif isinstance(event, bigframes.core.events.BigQueryRetryEvent):
@@ -226,8 +207,6 @@ def progress_callback(
elif isinstance(event, bigframes.core.events.BigQueryFinishedEvent):
message = render_bqquery_finished_event_plaintext(event)
print(message)
- elif isinstance(event, bigframes.core.events.ExecutionFinished):
- print("Execution done.")
def wait_for_job(job: GenericJob, progress_bar: Optional[str] = None):
diff --git a/notebooks/getting_started/magics.ipynb b/notebooks/getting_started/magics.ipynb
new file mode 100644
index 00000000000..1f2cf7a409b
--- /dev/null
+++ b/notebooks/getting_started/magics.ipynb
@@ -0,0 +1,406 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "91edcf7b",
+ "metadata": {},
+ "source": [
+ "# %%bqsql cell magics\n",
+ "\n",
+ "The BigQuery DataFrames (aka BigFrames) package provides a `%%bqsql` cell magics for Jupyter environments.\n",
+ "\n",
+ "To use it, first activate the extension:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "98cd0489",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%load_ext bigframes"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f18fdc63",
+ "metadata": {},
+ "source": [
+ "Now, use the magics by including SQL in the body."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "269c5862",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " Query processed 0 Bytes. [Job bigframes-dev:US.job_UVe7FsupxF3CbYuLcLT7fpw9dozg details]\n",
+ " "
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "1e2fb7b019754d31b11323a054f97f47",
+ "version_major": 2,
+ "version_minor": 1
+ },
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " state | \n",
+ " gender | \n",
+ " year | \n",
+ " name | \n",
+ " number | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " HI | \n",
+ " F | \n",
+ " 1999 | \n",
+ " Ariana | \n",
+ " 10 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " HI | \n",
+ " F | \n",
+ " 2002 | \n",
+ " Jordyn | \n",
+ " 10 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " HI | \n",
+ " F | \n",
+ " 2006 | \n",
+ " Mya | \n",
+ " 10 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " HI | \n",
+ " F | \n",
+ " 2010 | \n",
+ " Jordyn | \n",
+ " 10 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " HI | \n",
+ " M | \n",
+ " 1921 | \n",
+ " Nobuo | \n",
+ " 10 | \n",
+ "
\n",
+ " \n",
+ " | 5 | \n",
+ " HI | \n",
+ " M | \n",
+ " 1925 | \n",
+ " Ralph | \n",
+ " 10 | \n",
+ "
\n",
+ " \n",
+ " | 6 | \n",
+ " HI | \n",
+ " M | \n",
+ " 1926 | \n",
+ " Hisao | \n",
+ " 10 | \n",
+ "
\n",
+ " \n",
+ " | 7 | \n",
+ " HI | \n",
+ " M | \n",
+ " 1927 | \n",
+ " Moses | \n",
+ " 10 | \n",
+ "
\n",
+ " \n",
+ " | 8 | \n",
+ " HI | \n",
+ " M | \n",
+ " 1933 | \n",
+ " Larry | \n",
+ " 10 | \n",
+ "
\n",
+ " \n",
+ " | 9 | \n",
+ " HI | \n",
+ " M | \n",
+ " 1933 | \n",
+ " Alfredo | \n",
+ " 10 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
10 rows × 5 columns
\n",
+ "
[5552452 rows x 5 columns in total]"
+ ],
+ "text/plain": [
+ "state gender year name number\n",
+ " HI F 1999 Ariana 10\n",
+ " HI F 2002 Jordyn 10\n",
+ " HI F 2006 Mya 10\n",
+ " HI F 2010 Jordyn 10\n",
+ " HI M 1921 Nobuo 10\n",
+ " HI M 1925 Ralph 10\n",
+ " HI M 1926 Hisao 10\n",
+ " HI M 1927 Moses 10\n",
+ " HI M 1933 Larry 10\n",
+ " HI M 1933 Alfredo 10\n",
+ "...\n",
+ "\n",
+ "[5552452 rows x 5 columns]"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "%%bqsql\n",
+ "SELECT * FROM `bigquery-public-data.usa_names.usa_1910_2013`"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "8771e10f",
+ "metadata": {},
+ "source": [
+ "The output DataFrame can be saved to a variable."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "30bb6327",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " Query processed 0 Bytes. [Job bigframes-dev:US.c142adf3-cd95-42da-bbdc-c176b36b934f details]\n",
+ " "
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "%%bqsql mydf\n",
+ "SELECT * FROM `bigquery-public-data.usa_names.usa_1910_2013`"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "533e2e9e",
+ "metadata": {},
+ "source": [
+ "You can chain cells together using format strings. DataFrame objects are automatically turned into table expressions."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "6a8a8123",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " Query processed 88.1 MB in a moment of slot time.\n",
+ " "
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "c4889de9296440428de90defb5c58070",
+ "version_major": 2,
+ "version_minor": 1
+ },
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " total_count | \n",
+ " name | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 304036 | \n",
+ " Tracy | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 293876 | \n",
+ " Travis | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 203784 | \n",
+ " Troy | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 150127 | \n",
+ " Trevor | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 96397 | \n",
+ " Tristan | \n",
+ "
\n",
+ " \n",
+ " | 5 | \n",
+ " 89996 | \n",
+ " Tracey | \n",
+ "
\n",
+ " \n",
+ " | 6 | \n",
+ " 65546 | \n",
+ " Trinity | \n",
+ "
\n",
+ " \n",
+ " | 7 | \n",
+ " 50112 | \n",
+ " Traci | \n",
+ "
\n",
+ " \n",
+ " | 8 | \n",
+ " 49657 | \n",
+ " Trenton | \n",
+ "
\n",
+ " \n",
+ " | 9 | \n",
+ " 45692 | \n",
+ " Trent | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
10 rows × 2 columns
\n",
+ "
[238 rows x 2 columns in total]"
+ ],
+ "text/plain": [
+ " total_count name\n",
+ "0 304036 Tracy\n",
+ "1 293876 Travis\n",
+ "2 203784 Troy\n",
+ "3 150127 Trevor\n",
+ "4 96397 Tristan\n",
+ "5 89996 Tracey\n",
+ "6 65546 Trinity\n",
+ "7 50112 Traci\n",
+ "8 49657 Trenton\n",
+ "9 45692 Trent\n",
+ "...\n",
+ "\n",
+ "[238 rows x 2 columns]"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "%%bqsql\n",
+ "SELECT sum(number) as total_count, name\n",
+ "FROM {mydf}\n",
+ "WHERE name LIKE 'Tr%'\n",
+ "GROUP BY name\n",
+ "ORDER BY total_count DESC"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d2a17078",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "venv",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.18"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}