From 87910aa60df062a5299490923868821e4975c849 Mon Sep 17 00:00:00 2001 From: Grant Nestor Date: Sat, 4 Mar 2017 12:25:28 -0800 Subject: [PATCH 1/6] Use `df.to_json(orient='table'` --- jupyterlab_table/__init__.py | 7 +------ jupyterlab_table/utils.py | 17 +++++++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/jupyterlab_table/__init__.py b/jupyterlab_table/__init__.py index 1b06a49..c5c7e35 100755 --- a/jupyterlab_table/__init__.py +++ b/jupyterlab_table/__init__.py @@ -89,12 +89,7 @@ def schema(self, schema): def _ipython_display_(self): bundle = { 'application/vnd.dataresource+json': { - 'resources': [ - { - 'schema': self.schema, - 'data': prepare_data(self.data) - } - ] + 'resources': [prepare_data(self.data, self.schema)] }, 'text/plain': '' } diff --git a/jupyterlab_table/utils.py b/jupyterlab_table/utils.py index 427c309..24888fd 100644 --- a/jupyterlab_table/utils.py +++ b/jupyterlab_table/utils.py @@ -2,6 +2,7 @@ import codecs import collections import os.path +import json import pandas as pd @@ -72,10 +73,14 @@ def sanitize_dataframe(df): return df -def prepare_data(data=None): - """Prepare Plotly data from Pandas DataFrame.""" +def prepare_data(data=None, schema=None): + """Prepare JSONTable data from Pandas DataFrame.""" - if isinstance(data, list): - return data - data = sanitize_dataframe(data) - return data.to_dict(orient='records') + if isinstance(data, pd.DataFrame): + data = sanitize_dataframe(data) + data = data.to_json(orient='table') + return json.loads(data) + return { + 'data': data, + 'schema': schema + } From 54dc6bb5d06941759930095f5f9c4ed7ab42c48b Mon Sep 17 00:00:00 2001 From: Grant Nestor Date: Sun, 5 Mar 2017 15:08:47 -0800 Subject: [PATCH 2/6] Use `df.to_json` vs. `sanitize_dataframe` --- jupyterlab_table/__init__.py | 15 +++++-- jupyterlab_table/utils.py | 86 ------------------------------------ 2 files changed, 11 insertions(+), 90 deletions(-) delete mode 100644 jupyterlab_table/utils.py diff --git a/jupyterlab_table/__init__.py b/jupyterlab_table/__init__.py index c5c7e35..79ed5bc 100755 --- a/jupyterlab_table/__init__.py +++ b/jupyterlab_table/__init__.py @@ -1,7 +1,6 @@ from IPython.display import display, DisplayObject import json import pandas as pd -from .utils import prepare_data # Running `npm run build` will create static resources in the static @@ -22,6 +21,16 @@ def _jupyter_nbextension_paths(): 'require': 'jupyterlab_table/extension' }] +def prepare_data(data=None, schema=None): + """Prepare JSONTable data from Pandas DataFrame.""" + + if isinstance(data, pd.DataFrame): + data = data.to_json(orient='table') + return json.loads(data) + return { + 'data': data, + 'schema': schema + } # A display class that can be used within a notebook. E.g.: # from jupyterlab_table import JSONTable @@ -88,9 +97,7 @@ def schema(self, schema): def _ipython_display_(self): bundle = { - 'application/vnd.dataresource+json': { - 'resources': [prepare_data(self.data, self.schema)] - }, + 'application/vnd.dataresource+json': prepare_data(self.data, self.schema), 'text/plain': '' } metadata = { diff --git a/jupyterlab_table/utils.py b/jupyterlab_table/utils.py deleted file mode 100644 index 24888fd..0000000 --- a/jupyterlab_table/utils.py +++ /dev/null @@ -1,86 +0,0 @@ -import cgi -import codecs -import collections -import os.path -import json -import pandas as pd - - -def nested_update(d, u): - """Update nested dictionary d (in-place) with keys from u.""" - for k, v in u.items(): - if isinstance(v, collections.Mapping): - d[k] = nested_update(d.get(k, {}), v) - else: - d[k] = v - return d - - -def abs_path(path): - """Make path absolute.""" - return os.path.join( - os.path.dirname(os.path.abspath(__file__)), - path) - - -def get_content(path): - """Get content of file.""" - with codecs.open(abs_path(path), encoding='utf-8') as f: - return f.read() - - -def escape(string): - """Escape the string.""" - return cgi.escape(string, quote=True) - - -def sanitize_dataframe(df): - """Sanitize a DataFrame to prepare it for serialization. - - * Make a copy - * Raise ValueError if it has a hierarchical index. - * Convert categoricals to strings. - * Convert np.int dtypes to Python int objects - * Convert floats to objects and replace NaNs by None. - * Convert DateTime dtypes into appropriate string representations - """ - import pandas as pd - import numpy as np - - df = df.copy() - - if isinstance(df.index, pd.core.index.MultiIndex): - raise ValueError('Hierarchical indices not supported') - if isinstance(df.columns, pd.core.index.MultiIndex): - raise ValueError('Hierarchical indices not supported') - - for col_name, dtype in df.dtypes.iteritems(): - if str(dtype) == 'category': - # XXXX: work around bug in to_json for categorical types - # https://github.com/pydata/pandas/issues/10778 - df[col_name] = df[col_name].astype(str) - elif np.issubdtype(dtype, np.integer): - # convert integers to objects; np.int is not JSON serializable - df[col_name] = df[col_name].astype(object) - elif np.issubdtype(dtype, np.floating): - # For floats, convert nan->None: np.float is not JSON serializable - col = df[col_name].astype(object) - df[col_name] = col.where(col.notnull(), None) - elif str(dtype).startswith('datetime'): - # Convert datetimes to strings - # astype(str) will choose the appropriate resolution - df[col_name] = df[col_name].astype(str).replace('NaT', '') - return df - - -def prepare_data(data=None, schema=None): - """Prepare JSONTable data from Pandas DataFrame.""" - - if isinstance(data, pd.DataFrame): - data = sanitize_dataframe(data) - data = data.to_json(orient='table') - return json.loads(data) - return { - 'data': data, - 'schema': schema - } From 051d5f3b25ace96da976db47db750430b93605b0 Mon Sep 17 00:00:00 2001 From: Grant Nestor Date: Sun, 5 Mar 2017 15:09:08 -0800 Subject: [PATCH 3/6] Adopt new mimetype schema --- labextension/src/doc.js | 4 ++-- labextension/src/output.js | 8 ++++---- nbextension/src/renderer.js | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/labextension/src/doc.js b/labextension/src/doc.js index 5d739aa..28b6050 100755 --- a/labextension/src/doc.js +++ b/labextension/src/doc.js @@ -56,8 +56,8 @@ export class DocWidget extends Widget { if (this.isAttached) { const content = this._context.model.toString(); try { - const { resources: [ props ] } = JSON.parse(content); - ReactDOM.render(, this.node); + const { data, schema } = JSON.parse(content); + ReactDOM.render(, this.node); } catch (error) { const ErrorDisplay = props => (
, this.node); - // Inject static HTML into mime bundle + // if (data) + ReactDOM.render(, this.node); this._data.set( 'text/html', - ReactDOMServer.renderToStaticMarkup() + ReactDOMServer.renderToStaticMarkup() ); } } diff --git a/nbextension/src/renderer.js b/nbextension/src/renderer.js index 95ee948..03ad1e3 100644 --- a/nbextension/src/renderer.js +++ b/nbextension/src/renderer.js @@ -10,9 +10,9 @@ const CLASS_NAME = 'output_JSONTable rendered_html'; /** * Render data to the output area */ -function render(data, node) { - const { resources: [ props ] } = data; - ReactDOM.render(, node); +function render(json, node) { + const { data, schema } = json; + ReactDOM.render(, node); } /** From 2ea7402a331965c0e820c2d3160c94e8e3d0c159 Mon Sep 17 00:00:00 2001 From: Grant Nestor Date: Mon, 6 Mar 2017 12:13:40 -0800 Subject: [PATCH 4/6] Fix React invariant error --- component/virtualized-table.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/component/virtualized-table.js b/component/virtualized-table.js index 1dbc304..d0e0f9f 100644 --- a/component/virtualized-table.js +++ b/component/virtualized-table.js @@ -84,7 +84,7 @@ export default class VirtualizedTable extends React.Component { {this.schema.fields.map((field, fieldIndex) => ( // rowData // } From 151b2ad84c03bdde1188f0235aab32ccf91f057b Mon Sep 17 00:00:00 2001 From: Grant Nestor Date: Mon, 6 Mar 2017 12:14:01 -0800 Subject: [PATCH 5/6] Set pandas options in init --- jupyterlab_table/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/jupyterlab_table/__init__.py b/jupyterlab_table/__init__.py index 79ed5bc..0c92070 100755 --- a/jupyterlab_table/__init__.py +++ b/jupyterlab_table/__init__.py @@ -2,6 +2,9 @@ import json import pandas as pd +pd.options.display.html.table_schema = True +pd.options.display.max_rows = 10000 + # Running `npm run build` will create static resources in the static # directory of this Python package (and create that directory if necessary). From 2b6f4dd830df160fe86970833e7f7d75430b0702 Mon Sep 17 00:00:00 2001 From: Grant Nestor Date: Wed, 8 Mar 2017 15:41:40 -0800 Subject: [PATCH 6/6] Update virtualized-grid --- component/virtualized-grid.js | 38 ++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/component/virtualized-grid.js b/component/virtualized-grid.js index 14531a5..01e1969 100644 --- a/component/virtualized-grid.js +++ b/component/virtualized-grid.js @@ -1,5 +1,6 @@ import React from 'react'; -import { Grid, MultiGrid, AutoSizer, CellMeasurer } from 'react-virtualized'; +import ReactDOM from 'react-dom'; +import { MultiGrid, AutoSizer, CellMeasurer } from 'react-virtualized'; import 'react-virtualized/styles.css'; // hack: `stream.Transform` (stream-browserify) is undefined in `csv-parse` when // built with @jupyterlabextension-builder @@ -7,14 +8,20 @@ import infer from 'jsontableschema/lib/infer'; // import { infer } from 'jsontableschema'; import './index.css'; -const ROW_HEIGHT = 34; +const ROW_HEIGHT = 42; +const GRID_MAX_HEIGHT = 336; function inferSchema(data) { - const headers = data.reduce( - (result, row) => [ ...new Set([ ...result, ...Object.keys(row) ]) ], + // Take a sampling of rows from data + const range = Array.from({ length: 10 }, (v, i) => + Math.floor(Math.random() * data.length)); + // Separate headers and values + const headers = range.reduce( + (result, row) => [...new Set([...result, ...Object.keys(data[row])])], [] ); - const values = data.map(row => Object.values(row)); + const values = range.map(row => Object.values(data[row])); + // Infer column types and return schema for data return infer(headers, values); } @@ -32,6 +39,8 @@ export default class VirtualizedGrid extends React.Component { } render() { + const rowCount = this.data.length + 1 + const height = rowCount * ROW_HEIGHT; return ( {({ width }) => ( @@ -39,20 +48,21 @@ export default class VirtualizedGrid extends React.Component { cellRenderer={this.cellRenderer} columnCount={this.schema.fields.length} height={ROW_HEIGHT} - rowCount={this.data.length + 1} + rowCount={rowCount} > {({ getColumnWidth }) => ( getColumnWidth(index) + 10} + fixedColumnCount={1} fixedRowCount={1} height={ - (this.data.length + 1) * (ROW_HEIGHT + 8) < 400 - ? (this.data.length + 1) * (ROW_HEIGHT + 8) - : 400 + height < GRID_MAX_HEIGHT + ? height + : GRID_MAX_HEIGHT } - rowCount={this.data.length + 1} + rowCount={rowCount} rowHeight={ROW_HEIGHT} width={width} /> @@ -70,13 +80,13 @@ export default class VirtualizedGrid extends React.Component { style={{ ...style, fontFamily: '-apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"', - fontSize: 16, + fontSize: 12, fontWeight: rowIndex === 0 ? 600 : 'normal', backgroundColor: rowIndex % 2 === 0 && rowIndex !== 0 ? '#f8f8f8' : '#fff', - border: '1px solid #ddd', - padding: '6px 13px' + padding: '6px 13px', + boxSizing: 'border-box' }} > {