+ + {/* Header with title */} + + {toolName} + + Explore data with visualizations, powered by AI agents. + - - Explore data with visualizations, powered by AI agents. - {actionButtons} - {/* Interactive Features Carousel */} - - - {/* Left Arrow */} - + {features.map((feature, index) => ( + - - - - {/* Feature Content */} - {/* Text Content */} - {features[currentFeature].title} + {feature.title} - {features[currentFeature].description} + {feature.description} - - {/* Feature Indicators */} - - {features.map((_, index) => ( - setCurrentFeature(index)} - sx={{ - width: 32, - height: 4, - borderRadius: 2, - bgcolor: index === currentFeature - ? theme.palette.primary.main - : alpha(theme.palette.text.secondary, 0.2), - cursor: 'pointer', - '&:hover': { - bgcolor: index === currentFeature - ? theme.palette.primary.main - : alpha(theme.palette.text.secondary, 0.4), - } - }} - /> - ))} - {/* Media Content */} @@ -292,30 +177,18 @@ export const About: FC<{}> = function About({ }) { flex: 1, borderRadius: 2, overflow: 'hidden', - boxShadow: '0 4px 12px rgba(0,0,0,0.1)', - minWidth: 300, - maxWidth: 500, + border: '1px solid rgba(0,0,0,0.1)', }}> - {features[currentFeature].mediaType === 'video' ? ( + {feature.mediaType === 'video' ? ( { - const video = e.currentTarget as HTMLVideoElement; - if (video.duration && !isNaN(video.duration)) { - videoDurationsRef.current.set( - features[currentFeature].media, - video.duration - ); - } - }} + aria-label={`Video demonstration: ${feature.title}`} sx={{ width: '100%', height: 'auto', @@ -325,8 +198,8 @@ export const About: FC<{}> = function About({ }) { ) : ( = function About({ }) { )} - - {/* Right Arrow */} - - - - - - - {/* Screenshots Carousel Section */} - - - {/* Screenshot Container */} - setCurrentScreenshot((currentScreenshot + 1) % screenshots.length)} - sx={{ - height: 680, - width: 'auto', - borderRadius: 8, - cursor: 'pointer', - overflow: 'hidden', - border: '1px solid rgba(0,0,0,0.1)', - boxShadow: '0 4px 12px rgba(0,0,0,0.1)', - position: 'relative', - display: 'flex', - justifyContent: 'center', - textDecoration: 'none', - animation: 'fadeSlideIn 0.1s ease-out', - '&:hover': { - boxShadow: '0 8px 24px rgba(0,0,0,0.2)', - '& .description-overlay': { - opacity: 1, - } - } - }} - > - - - - {screenshots[currentScreenshot].description} - - - - - {/* Screenshot Indicators */} - - {screenshots.map((_, index) => ( - setCurrentScreenshot(index)} - sx={{ - width: 32, - height: 4, - borderRadius: 2, - bgcolor: index === currentScreenshot - ? theme.palette.primary.main - : alpha(theme.palette.text.secondary, 0.2), - cursor: 'pointer', - transition: 'all 0.3s ease', - '&:hover': { - bgcolor: index === currentScreenshot - ? theme.palette.primary.main - : alpha(theme.palette.text.secondary, 0.4), - } - }} - /> - ))} - - + ))} - - - How does Data Formulator handle your data? - - -
  • Data Storage: Uploaded data (csv, xlsx, json, clipboard, messy data etc.) is stored in browser's local storage only
  • -
  • Data Processing: Local installation runs Python on your machine; online demo sends the data to server for data transformations (but not stored)
  • -
  • Database: Only available for locally installed Data Formulator (a DuckDB database file is created in temp directory to store data); not available in online demo
  • -
  • LLM Endpoints: Small data samples are sent to LLM endpoints along with the prompt. Use your trusted model provider if working with private data.
  • -
    - - Research Prototype from Microsoft Research - -
    + + Data handling: Data stored in browser only • Local install runs Python locally; online demo processes server-side (not stored) • LLM receives small samples with prompts + + + Research Prototype from Microsoft Research +
    {/* Footer */} - + - + ) } diff --git a/src/views/ChartRecBox.tsx b/src/views/ChartRecBox.tsx index 943f833..850631b 100644 --- a/src/views/ChartRecBox.tsx +++ b/src/views/ChartRecBox.tsx @@ -69,111 +69,6 @@ export interface ChartRecBoxProps { sx?: SxProps; } -// Table selector component for ChartRecBox -const NLTableSelector: FC<{ - selectedTableIds: string[], - tables: DictTable[], - updateSelectedTableIds: (tableIds: string[]) => void, - requiredTableIds?: string[] -}> = ({ selectedTableIds, tables, updateSelectedTableIds, requiredTableIds = [] }) => { - const [anchorEl, setAnchorEl] = useState(null); - const open = Boolean(anchorEl); - - const handleClick = (event: React.MouseEvent) => { - setAnchorEl(event.currentTarget); - }; - - const handleClose = () => { - setAnchorEl(null); - }; - - const handleTableSelect = (table: DictTable) => { - if (!selectedTableIds.includes(table.id)) { - updateSelectedTableIds([...selectedTableIds, table.id]); - } - handleClose(); - }; - - return ( - - {selectedTableIds.map((tableId) => { - const isRequired = requiredTableIds.includes(tableId); - return ( - t.id == tableId)?.displayId} - size="small" - sx={{ - height: 16, - fontSize: '10px', - borderRadius: '2px', - bgcolor: isRequired ? 'rgba(25, 118, 210, 0.2)' : 'rgba(25, 118, 210, 0.1)', - color: 'rgba(0, 0, 0, 0.7)', - '& .MuiChip-label': { - pl: '4px', - pr: '6px' - } - }} - deleteIcon={isRequired ? undefined : } - onDelete={isRequired ? undefined : () => updateSelectedTableIds(selectedTableIds.filter(id => id !== tableId))} - /> - ); - })} - - - - - - - {tables - .filter(t => t.derive === undefined || t.anchored) - .map((table) => { - const isSelected = selectedTableIds.includes(table.id); - const isRequired = requiredTableIds.includes(table.id); - return ( - handleTableSelect(table)} - sx={{ - fontSize: '12px', - display: 'flex', - justifyContent: 'space-between', - alignItems: 'center' - }} - > - {table.displayId} - {isRequired && (required)} - - ); - }) - } - - - ); -}; - - - export const IdeaChip: FC<{ mini?: boolean, idea: {text?: string, questions?: string[], goal: string, difficulty: 'easy' | 'medium' | 'hard', type?: 'branch' | 'deep_dive'} @@ -357,17 +252,11 @@ export const ChartRecBox: FC = function ({ tableId, placeHolde const currentTable = tables.find(t => t.id === tableId); const availableTables = tables.filter(t => t.derive === undefined || t.anchored); - const [additionalTableIds, setAdditionalTableIds] = useState([]); - + // Combine the main tableId with additional selected tables - const selectedTableIds = currentTable?.derive ? [...currentTable.derive.source, ...additionalTableIds] : [tableId, ...additionalTableIds]; - - const handleTableSelectionChange = (newTableIds: string[]) => { - // Filter out the main tableId since it's always included - const additionalIds = newTableIds.filter(id => id !== tableId); - setAdditionalTableIds(additionalIds); - }; - + let selectedTableIds = currentTable?.derive ? [...currentTable.derive.source] : [tableId]; + selectedTableIds = [...selectedTableIds, ...availableTables.map(t => t.id).filter(id => !selectedTableIds.includes(id))]; + // Function to get a question from the list with cycling const getQuestion = (): string => { return mode === "agent" ? "let's explore something interesting about the data" : "show something interesting about the data"; @@ -1238,17 +1127,6 @@ export const ChartRecBox: FC = function ({ tableId, placeHolde }} /> )} - {showTableSelector && ( - - - - )} - = function ({ tableId, placeHolde "& .MuiInput-underline:not(.Mui-disabled):before": { borderBottom: 'none', }, - "& .MuiInput-underline:(.Mui-disabled):before": { + "& .MuiInput-underline.Mui-disabled:before": { + borderBottom: 'none', + }, + "& .MuiInput-underline.Mui-disabled:after": { borderBottom: 'none', }, "& .MuiInput-underline:after": { diff --git a/src/views/DBTableManager.tsx b/src/views/DBTableManager.tsx index da5bc8c..8d09016 100644 --- a/src/views/DBTableManager.tsx +++ b/src/views/DBTableManager.tsx @@ -1,5 +1,5 @@ // TableManager.tsx -import React, { useState, useEffect, FC } from 'react'; +import React, { useState, useEffect, useCallback, FC, useRef } from 'react'; import { Card, CardContent, @@ -9,13 +9,7 @@ import { Box, IconButton, Paper, - Tabs, - Tab, TextField, - Dialog, - DialogActions, - DialogContent, - DialogTitle, Divider, SxProps, Table, @@ -26,29 +20,29 @@ import { TableRow, CircularProgress, ButtonGroup, + ToggleButton, + ToggleButtonGroup, Tooltip, MenuItem, + Menu, Chip, Collapse, styled, - ToggleButtonGroup, - ToggleButton, useTheme, Link, - Checkbox + Checkbox, + Popover } from '@mui/material'; import DeleteIcon from '@mui/icons-material/Delete'; import CloseIcon from '@mui/icons-material/Close'; -import AnalyticsIcon from '@mui/icons-material/Analytics'; import ExpandLessIcon from '@mui/icons-material/ExpandLess'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import TableRowsIcon from '@mui/icons-material/TableRows'; import RefreshIcon from '@mui/icons-material/Refresh'; -import ArrowForwardIcon from '@mui/icons-material/ArrowForward'; import SearchIcon from '@mui/icons-material/Search'; -import { getUrls } from '../app/utils'; +import { getUrls, fetchWithSession } from '../app/utils'; import { CustomReactTable } from './ReactTable'; import { DictTable } from '../components/ComponentType'; import { Type } from '../data/types'; @@ -58,27 +52,68 @@ import { alpha } from '@mui/material'; import { DataFormulatorState } from '../app/dfSlice'; import { fetchFieldSemanticType } from '../app/dfSlice'; import { AppDispatch } from '../app/store'; -import Editor from 'react-simple-code-editor'; import Markdown from 'markdown-to-jsx'; -import Prism from 'prismjs' -import 'prismjs/components/prism-javascript' // Language -import 'prismjs/themes/prism.css'; //Example style, you can use another -import PrecisionManufacturingIcon from '@mui/icons-material/PrecisionManufacturing'; import CheckIcon from '@mui/icons-material/Check'; import MuiMarkdown from 'mui-markdown'; import CleaningServicesIcon from '@mui/icons-material/CleaningServices'; +import MoreVertIcon from '@mui/icons-material/MoreVert'; +import UploadFileIcon from '@mui/icons-material/UploadFile'; +import DownloadIcon from '@mui/icons-material/Download'; +import RestartAltIcon from '@mui/icons-material/RestartAlt'; +import StorageIcon from '@mui/icons-material/Storage'; +import CloudUploadIcon from '@mui/icons-material/CloudUpload'; +import SettingsIcon from '@mui/icons-material/Settings'; -export const handleDBDownload = async (sessionId: string) => { +// Industry-standard database icons +const TableIcon: React.FC<{ sx?: SxProps }> = ({ sx }) => ( + + {/* Single rectangle with grid lines - standard table icon */} + + + + + + +); + +const ViewIcon: React.FC<{ sx?: SxProps }> = ({ sx }) => ( + + {/* Two overlapping rectangles - standard view icon */} + + + +); + +export const handleDBDownload = async (sessionId: string, dispatch?: any) => { try { - const response = await fetch(getUrls().DOWNLOAD_DB_FILE, { - method: 'GET', - }); + // Use fetchWithSession which automatically handles session ID if missing + const response = await fetchWithSession( + getUrls().DOWNLOAD_DB_FILE, + { method: 'GET' }, + dispatch + ); // Check if the response is ok if (!response.ok) { const errorData = await response.json(); - throw new Error(errorData.error || 'Failed to download database file'); + throw new Error(errorData.error || errorData.message || 'Failed to download database file'); } // Get the blob directly from response @@ -88,7 +123,7 @@ export const handleDBDownload = async (sessionId: string) => { // Create a temporary link element const link = document.createElement('a'); link.href = url; - link.download = `df_${sessionId?.slice(0, 4)}.db`; + link.download = `df_${sessionId?.slice(0, 4) || 'db'}.db`; document.body.appendChild(link); // Trigger download @@ -126,116 +161,9 @@ interface ColumnStatistics { }; } -interface TableStatisticsViewProps { - tableName: string; - columnStats: ColumnStatistics[]; -} - -export class TableStatisticsView extends React.Component { - render() { - const { tableName, columnStats } = this.props; - - // Common styles for header cells - const headerCellStyle = { - backgroundColor: '#fff', - fontSize: 10, - color: "#333", - borderBottomColor: (theme: any) => theme.palette.primary.main, - borderBottomWidth: '1px', - borderBottomStyle: 'solid', - padding: '6px' - }; - - // Common styles for body cells - const bodyCellStyle = { - fontSize: 10, - padding: '6px' - }; - - return ( - - - - - - Column - Type - Count - Unique - Null - Min - Max - Avg - - - - {columnStats.map((stat, idx) => ( - - - {stat.column} - - - {stat.type} - - - {stat.statistics.count} - - - {stat.statistics.unique_count} - - - {stat.statistics.null_count} - - - {stat.statistics.min !== undefined ? stat.statistics.min : '-'} - - - {stat.statistics.max !== undefined ? stat.statistics.max : '-'} - - - {stat.statistics.avg !== undefined ? - Number(stat.statistics.avg).toFixed(2) : '-'} - - - ))} - -
    -
    -
    - ); - } -} -export const DBTableSelectionDialog: React.FC<{ - buttonElement?: any, - sx?: SxProps, - onOpen?: () => void, - // Controlled mode props - open?: boolean, - onClose?: () => void -}> = function DBTableSelectionDialog({ - buttonElement, - sx, - onOpen, - open: controlledOpen, - onClose, -}) { +export const DBManagerPane: React.FC<{ +}> = function DBManagerPane({ }) { const theme = useTheme(); @@ -243,15 +171,10 @@ export const DBTableSelectionDialog: React.FC<{ const sessionId = useSelector((state: DataFormulatorState) => state.sessionId); const tables = useSelector((state: DataFormulatorState) => state.tables); const serverConfig = useSelector((state: DataFormulatorState) => state.serverConfig); + const dataLoaderConnectParams = useSelector((state: DataFormulatorState) => state.dataLoaderConnectParams); - const [internalOpen, setInternalOpen] = useState(false); // Support both controlled and uncontrolled modes - const isControlled = controlledOpen !== undefined; - const tableDialogOpen = isControlled ? controlledOpen : internalOpen; - const setTableDialogOpen = isControlled - ? (open: boolean) => { if (!open && onClose) onClose(); } - : setInternalOpen; const [tableAnalysisMap, setTableAnalysisMap] = useState>({}); // maps data loader type to list of param defs @@ -261,8 +184,15 @@ export const DBTableSelectionDialog: React.FC<{ const [dbTables, setDbTables] = useState([]); const [selectedTabKey, setSelectedTabKey] = useState(""); + const [selectedDataLoader, setSelectedDataLoader] = useState(""); + const [connectorMenuAnchorEl, setConnectorMenuAnchorEl] = useState(null); const [isUploading, setIsUploading] = useState(false); + const [resetAnchorEl, setResetAnchorEl] = useState(null); + const [tableMenuAnchorEl, setTableMenuAnchorEl] = useState(null); + const [showViews, setShowViews] = useState(false); + const dbFileInputRef = useRef(null); + const menuButtonRef = useRef(null); let setSystemMessage = (content: string, severity: "error" | "warning" | "info" | "success") => { dispatch(dfActions.addMessages({ @@ -278,18 +208,20 @@ export const DBTableSelectionDialog: React.FC<{ }, []); useEffect(() => { - if (!selectedTabKey.startsWith("dataLoader:") && dbTables.length == 0) { - setSelectedTabKey(""); - } else if (!selectedTabKey.startsWith("dataLoader:") && dbTables.find(t => t.name === selectedTabKey) == undefined) { - setSelectedTabKey(dbTables[0].name); + if (selectedDataLoader === "") { + if (dbTables.length == 0) { + setSelectedTabKey(""); + } else if (dbTables.find(t => t.name === selectedTabKey) == undefined) { + setSelectedTabKey(dbTables[0]?.name || ""); + } } - }, [dbTables]); + }, [dbTables, selectedDataLoader]); // Fetch list of tables const fetchTables = async () => { if (serverConfig.DISABLE_DATABASE) return; try { - const response = await fetch(getUrls().LIST_TABLES); + const response = await fetchWithSession(getUrls().LIST_TABLES, { method: 'GET' }, dispatch); const data = await response.json(); if (data.status === 'success') { setDbTables(data.tables); @@ -329,10 +261,10 @@ export const DBTableSelectionDialog: React.FC<{ try { setIsUploading(true); - const response = await fetch(getUrls().UPLOAD_DB_FILE, { + const response = await fetchWithSession(getUrls().UPLOAD_DB_FILE, { method: 'POST', body: formData - }); + }, dispatch); const data = await response.json(); if (data.status === 'success') { fetchTables(); // Refresh table list @@ -357,10 +289,10 @@ export const DBTableSelectionDialog: React.FC<{ try { setIsUploading(true); - const response = await fetch(getUrls().CREATE_TABLE, { + const response = await fetchWithSession(getUrls().CREATE_TABLE, { method: 'POST', body: formData - }); + }, dispatch); const data = await response.json(); if (data.status === 'success') { if (data.is_renamed) { @@ -383,9 +315,9 @@ export const DBTableSelectionDialog: React.FC<{ const handleDBReset = async () => { try { - const response = await fetch(getUrls().RESET_DB_FILE, { + const response = await fetchWithSession(getUrls().RESET_DB_FILE, { method: 'POST', - }); + }, dispatch); const data = await response.json(); if (data.status === 'success') { fetchTables(); @@ -405,13 +337,13 @@ export const DBTableSelectionDialog: React.FC<{ let deletedViews = []; for (let view of unreferencedViews) { try { - const response = await fetch(getUrls().DELETE_TABLE, { + const response = await fetchWithSession(getUrls().DELETE_TABLE, { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify({ table_name: view.name }) - }); + }, dispatch); const data = await response.json(); if (data.status === 'success') { deletedViews.push(view.name); @@ -438,13 +370,13 @@ export const DBTableSelectionDialog: React.FC<{ } try { - const response = await fetch(getUrls().DELETE_TABLE, { + const response = await fetchWithSession(getUrls().DELETE_TABLE, { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify({ table_name: tableName }) - }); + }, dispatch); const data = await response.json(); if (data.status === 'success') { fetchTables(); @@ -457,24 +389,21 @@ export const DBTableSelectionDialog: React.FC<{ } }; - // Handle data analysis - const handleAnalyzeData = async (tableName: string) => { + // Handle data analysis - auto-fetch when table is selected + const handleAnalyzeData = useCallback(async (tableName: string) => { if (!tableName) return; if (tableAnalysisMap[tableName]) return; - console.log('Analyzing table:', tableName); - try { - const response = await fetch(getUrls().GET_COLUMN_STATS, { + const response = await fetchWithSession(getUrls().GET_COLUMN_STATS, { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify({ table_name: tableName }) - }); + }, dispatch); const data = await response.json(); if (data.status === 'success') { - console.log('Analysis results:', data); // Update the analysis map with the new results setTableAnalysisMap(prevMap => ({ ...prevMap, @@ -483,24 +412,16 @@ export const DBTableSelectionDialog: React.FC<{ } } catch (error) { console.error('Failed to analyze table data:', error); - setSystemMessage('Failed to analyze table data, please check if the server is running', "error"); + // Don't show error message for auto-fetch, just fail silently } - }; + }, [tableAnalysisMap]); - // Toggle analysis view - const toggleAnalysisView = (tableName: string) => { - if (tableAnalysisMap[tableName]) { - // If we already have analysis, remove it to show table data again - setTableAnalysisMap(prevMap => { - const newMap = { ...prevMap }; - delete newMap[tableName]; - return newMap; - }); - } else { - // If no analysis yet, fetch it - handleAnalyzeData(tableName); + // Auto-fetch stats when a table is selected + useEffect(() => { + if (selectedTabKey && selectedDataLoader === "") { + handleAnalyzeData(selectedTabKey); } - }; + }, [selectedTabKey, selectedDataLoader, handleAnalyzeData]); const handleAddTableToDF = (dbTable: DBTable) => { const convertSqlTypeToAppType = (sqlType: string): Type => { @@ -542,44 +463,12 @@ export const DBTableSelectionDialog: React.FC<{ } dispatch(dfActions.loadTable(table)); dispatch(fetchFieldSemanticType(table)); - setTableDialogOpen(false); } - const handleTabChange = (event: React.SyntheticEvent, newValue: string) => { - setSelectedTabKey(newValue); - }; useEffect(() => { - if (tableDialogOpen) { - fetchTables(); - } - }, [tableDialogOpen]); - - let importButton = (buttonElement: React.ReactNode) => { - return - - - - - } - - let exportButton = - - - - - + fetchTables(); + }, []); function uploadFileButton(element: React.ReactNode, buttonSx?: SxProps) { return ( @@ -605,81 +494,182 @@ export const DBTableSelectionDialog: React.FC<{ ); } - let hasDerivedViews = dbTables.filter(t => t.view_source !== null).length > 0; - - let dataLoaderPanel = - + let tableSelectionPanel = + {/* Recent Data Loaders */} + + + External Data Loaders + + + setSelectedDataLoader("file upload")} + sx={{ + fontSize: '0.7rem', + height: 20, + maxWidth: '100%', + borderColor: selectedDataLoader === "file upload" + ? theme.palette.secondary.main + : theme.palette.divider, + '& .MuiChip-label': { + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap' + }, + }} + /> + {Object.keys(dataLoaderMetadata ?? {}) + .map((dataLoaderType) => ( + setSelectedDataLoader(dataLoaderType)} + sx={{ + fontSize: '0.7rem', + height: 20, + maxWidth: '100%', + backgroundColor: selectedDataLoader === dataLoaderType + ? alpha(theme.palette.secondary.main, 0.2) + : 'transparent', + borderColor: selectedDataLoader === dataLoaderType + ? theme.palette.secondary.main + : theme.palette.divider, + '& .MuiChip-label': { + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap' + }, + }} + /> + ))} + + + + - Data Connectors + Local DuckDB - - - {["file upload", ...Object.keys(dataLoaderMetadata ?? {})].map((dataLoaderType, i) => ( - - ))} - ; - - let tableSelectionPanel = - - - Data Tables - - - { - fetchTables(); - }}> - - - + { + fetchTables(); + setTableMenuAnchorEl(null); + }} + dense + > + + Refresh table list + + + { + dbFileInputRef.current?.click(); + setTableMenuAnchorEl(null); + }} + disabled={isUploading} + dense + > + + Import database file + + { + if (!isUploading && dbTables.length > 0) { + // fetchWithSession will automatically handle session ID if missing + handleDBDownload(sessionId ?? '', dispatch) + .catch(error => { + console.error('Failed to download database:', error); + setSystemMessage('Failed to download database file', "error"); + }); + } + setTableMenuAnchorEl(null); + }} + disabled={isUploading || dbTables.length === 0} + dense + > + + Export database file + + { + setTableMenuAnchorEl(null); + if (!isUploading && menuButtonRef.current) { + // Use setTimeout to ensure menu closes before popover opens + setTimeout(() => { + setResetAnchorEl(menuButtonRef.current); + }, 100); + } + }} + disabled={isUploading} + dense + sx={{ color: 'error.main' }} + > + + Reset database + + + + {dbTables.length == 0 && @@ -688,122 +678,165 @@ export const DBTableSelectionDialog: React.FC<{ } {/* Regular Tables */} - {dbTables.filter(t => t.view_source === null).map((t, i) => ( - - ))} + textTransform: "none", + width: '100%', + maxWidth: '100%', + justifyContent: 'flex-start', + textAlign: 'left', + borderRadius: 0, + py: 0.5, + px: 2, + color: (selectedTabKey === t.name && selectedDataLoader === "") ? 'primary.main' : 'text.secondary', + borderRight: (selectedTabKey === t.name && selectedDataLoader === "") ? 2 : 0, + minWidth: 0, + }} + startIcon={} + endIcon={isLoaded ? : null} + > + + {t.name} + + + ); + })} {/* Derived Views Section */} - {hasDerivedViews && ( + {dbTables.filter(t => t.view_source !== null).length > 0 && ( - - - Derived Views - - - t.view_source !== null).length === 0} - onClick={() => { - handleCleanDerivedViews(); - }}> - - - - - - {dbTables.filter(t => t.view_source !== null).map((t, i) => ( + - ))} + + t.view_source !== null && t.view_source !== undefined && !tables.some(t2 => t2.id === t.name)).length === 0} + sx={{ + padding: 0.5, + mr: 0.5, + '&:hover': { + backgroundColor: alpha(theme.palette.primary.main, 0.08), + } + }} + > + + + + + + + {dbTables.filter(t => t.view_source !== null).map((t, i) => { + return ( + + ); + })} + + )} - let tableView = - {/* Empty state */} - {selectedTabKey === '' && ( - - The database is empty, refresh the table list or import some data to get started. - - )} + let dataConnectorView = + {/* File upload */} - {selectedTabKey === 'dataLoader:file upload' && ( + {selectedDataLoader === 'file upload' && ( {uploadFileButton({isUploading ? 'uploading...' : 'upload a csv/tsv file to the local database'})} @@ -811,7 +844,7 @@ export const DBTableSelectionDialog: React.FC<{ {/* Data loader forms */} {dataLoaderMetadata && Object.entries(dataLoaderMetadata).map(([dataLoaderType, metadata]) => ( - selectedTabKey === 'dataLoader:' + dataLoaderType && ( + selectedDataLoader === dataLoaderType && ( { setIsUploading(false); fetchTables().then(() => { + // Switch back to tables view after import + setSelectedDataLoader(""); // Navigate to the first imported table after tables are fetched if (status === "success" && importedTables && importedTables.length > 0) { setSelectedTabKey(importedTables[0]); @@ -837,196 +872,253 @@ export const DBTableSelectionDialog: React.FC<{ ) ))} + ; + + let tableView = + {/* Empty state */} + {selectedTabKey === '' && ( + + The database is empty, refresh the table list or import some data to get started. + + )} {/* Table content */} {dbTables.map((t, i) => { if (selectedTabKey !== t.name) return null; const currentTable = t; - const showingAnalysis = tableAnalysisMap[currentTable.name] !== undefined; + const columnStats = tableAnalysisMap[currentTable.name] ?? []; + const statsMap = new Map(columnStats.map((stat: ColumnStatistics) => [stat.column, stat])); + return ( - - - - {showingAnalysis ? "column stats for " : "sample data from "} - - {currentTable.name} - - - ({currentTable.columns.length} columns × {currentTable.row_count} rows) - + + + {currentTable.view_source ? : } + + {currentTable.name} + + + + handleDropTable(currentTable.name)} + title="Drop Table" + > + + + + + + + + + + + {currentTable.columns.map((col) => { + const stat = statsMap.get(col.name); + return ( + + + + {col.name} + + {stat ? ( + + {stat.type} + + ) : ( + + {col.type} + + )} + + + ); + })} + + + + {currentTable.sample_rows.slice(0, 9).map((row: any, rowIdx: number) => ( + + {currentTable.columns.map((col) => { + const value = row[col.name]; + return ( + + {String(value ?? '')} + + ); + })} + + ))} + +
    +
    + {currentTable.row_count > 10 && ( + + + Showing first 9 rows of {currentTable.row_count} total rows + + + )} +
    +
    + {tables.some(t => t.id === currentTable.name) ? ( + + + + Loaded - - + )} + + ); + })} + ; + + let mainContent = + + + {/* Button navigation - similar to TableSelectionView */} + + + {/* Available Tables Section - always visible */} + {tableSelectionPanel} + + {/* Reset Confirmation Popover */} + setResetAnchorEl(null)} + anchorOrigin={{ + vertical: 'bottom', + horizontal: 'left', + }} + transformOrigin={{ + vertical: 'top', + horizontal: 'left', + }} + > + + + Reset backend database and delete all tables? This cannot be undone. + + + - handleDropTable(currentTable.name)} - title="Drop Table" + variant="contained" + onClick={async () => { + setResetAnchorEl(null); + await handleDBReset(); + }} + sx={{ textTransform: 'none', fontSize: '12px', minWidth: 'auto', px: 0.75, py: 0.25, minHeight: '24px' }} > - - + Reset + - {showingAnalysis ? ( - - ) : ( - { - return Object.fromEntries(Object.entries(row).map(([key, value]: [string, any]) => { - return [key, String(value)]; - })); - }).slice(0, 9)} - columnDefs={currentTable.columns.map(col => ({ - id: col.name, - label: col.name, - minWidth: 60 - }))} - rowsPerPageNum={-1} - compact={false} - isIncompleteTable={currentTable.row_count > 10} - /> - )} -
    - +
    - ); - })} -
    ; - - let mainContent = - - {/* Button navigation - similar to TableSelectionView */} - - - {/* External Data Loaders Section */} - {dataLoaderPanel} - {/* Available Tables Section */} - {tableSelectionPanel} + {/* Content area - show connector view if a connector is selected, otherwise show table view */} + + {selectedDataLoader !== "" ? dataConnectorView : tableView} - - {importButton(Import)} - , - {exportButton} - or - - the backend database - - {/* Content area - using conditional rendering instead of TabPanel */} - {tableView} return ( - <> - {buttonElement && ( - - Install Data Formulator locally to use database.
    - Link: e.stopPropagation()} - > - https://github.com/microsoft/data-formulator - - - ) : ""} - placement="top" - > - - - -
    + + {mainContent} + {isUploading && ( + + + )} - {setTableDialogOpen(false)}} - open={tableDialogOpen} - sx={{ '& .MuiDialog-paper': { maxWidth: '100%', maxHeight: 800, minWidth: 800 } }} - > - - Database - setTableDialogOpen(false)} - > - - - - - {mainContent} - {isUploading && ( - - - - )} - - - + ); } @@ -1047,46 +1139,14 @@ export const DataLoaderForm: React.FC<{ let [tableFilter, setTableFilter] = useState(""); const [selectedTables, setSelectedTables] = useState>(new Set()); - const [displayAuthInstructions, setDisplayAuthInstructions] = useState(false); - let [isConnecting, setIsConnecting] = useState(false); - let [mode, setMode] = useState<"view tables" | "query">("view tables"); const toggleDisplaySamples = (tableName: string) => { setDisplaySamples({...displaySamples, [tableName]: !displaySamples[tableName]}); } - const handleModeChange = (event: React.MouseEvent, newMode: "view tables" | "query") => { - if (newMode != null) { - setMode(newMode); - } - }; - let tableMetadataBox = [ - - - View Tables - Query Data - - - , - mode === "view tables" && + {Object.entries(tableMetadata).map(([tableName, metadata]) => { @@ -1095,17 +1155,32 @@ export const DataLoaderForm: React.FC<{ key={tableName} sx={{ '&:last-child td, &:last-child th': { border: 0 }, - '& .MuiTableCell-root': { padding: 0.25, wordWrap: 'break-word', whiteSpace: 'normal' }, + '& .MuiTableCell-root': { + borderBottom: displaySamples[tableName] ? 'none' : '1px solid rgba(0, 0, 0, 0.1)', + padding: 0.25, wordWrap: 'break-word', whiteSpace: 'normal'}, backgroundColor: selectedTables.has(tableName) ? 'action.selected' : 'inherit', - '&:hover': { backgroundColor: selectedTables.has(tableName) ? 'action.selected' : 'action.hover' } + '&:hover': { backgroundColor: selectedTables.has(tableName) ? 'action.selected' : 'action.hover' }, + cursor: 'pointer', + }} + onClick={() => { + const newSelected = new Set(selectedTables); + if (newSelected.has(tableName)) { + newSelected.delete(tableName); + } else { + newSelected.add(tableName); + } + setSelectedTables(newSelected); }} > - - toggleDisplaySamples(tableName)}> + + { + e.stopPropagation(); + toggleDisplaySamples(tableName); + }}> {displaySamples[tableName] ? : } - + {tableName} ({metadata.row_count > 0 ? `${metadata.row_count} rows × ` : ""}{metadata.columns.length} cols) @@ -1115,19 +1190,10 @@ export const DataLoaderForm: React.FC<{ ))} - + { - const newSelected = new Set(selectedTables); - if (e.target.checked) { - newSelected.add(tableName); - } else { - newSelected.delete(tableName); - } - setSelectedTables(newSelected); - }} /> , @@ -1135,7 +1201,7 @@ export const DataLoaderForm: React.FC<{ - + { return Object.fromEntries(Object.entries(row).map(([key, value]: [string, any]) => { return [key, String(value)]; @@ -1146,7 +1212,7 @@ export const DataLoaderForm: React.FC<{ compact={false} isIncompleteTable={metadata.row_count > 10} /> - + ] @@ -1154,18 +1220,20 @@ export const DataLoaderForm: React.FC<{
    , - mode === "view tables" && Object.keys(tableMetadata).length > 0 && + Object.keys(tableMetadata).length > 0 && - , - mode === "query" && ({name: t, fields: tableMetadata[t].columns.map((c: any) => c.name)}))} - dataLoaderParams={params} onImport={onImport} onFinish={onFinish} /> + ] + const isConnected = Object.keys(tableMetadata).length > 0; + return ( + + Import tables from {dataLoaderType} + {isConnecting && } - - Data Connector ({dataLoaderType}) - - - {paramDefs.map((paramDef) => ( - - 0} - sx={{width: "270px", - '& .MuiInputLabel-root': {fontSize: 14}, - '& .MuiInputBase-root': {fontSize: 14}, - '& .MuiInputBase-input::placeholder': {fontSize: 12, fontStyle: "italic"} - }} - variant="standard" - size="small" - required={paramDef.required} - key={paramDef.name} - label={paramDef.name} - value={params[paramDef.name] ?? ''} - placeholder={paramDef.default ? `e.g. ${paramDef.default}` : paramDef.description} - onChange={(event: any) => { - dispatch(dfActions.updateDataLoaderConnectParam({ - dataLoaderType, paramName: paramDef.name, - paramValue: event.target.value})); - }} - slotProps={{ - inputLabel: {shrink: true} - }} - /> + {isConnected ? ( + // Connected state: show connection parameters and disconnect button + + + + + {paramDefs.filter((paramDef) => params[paramDef.name]).map((paramDef, index) => ( + + + {paramDef.name}: + + + {params[paramDef.name] || '(empty)'} + + {index < paramDefs.filter((paramDef) => params[paramDef.name]).length - 1 && ( + + • + + )} + + ))} + + + + + + + + table filter + + + setTableFilter(event.target.value)} + /> + + + + + + - ))} - - - table filter - } - placeholder="load only tables containing keywords" - value={tableFilter} - onChange={(event) => setTableFilter(event.target.value)} - slotProps={{ - inputLabel: {shrink: true}, - }} - /> - {paramDefs.length > 0 && - - - } - - - - - - - { - - + .catch((error: any) => { + onFinish("error", `Failed to fetch data loader tables, please check the server is running`); + setIsConnecting(false); + }); + }}> + connect {tableFilter.trim() ? "with filter" : ""} + + } + + + + + {authInstructions.trim()} - - - } - - {Object.keys(tableMetadata).length > 0 && tableMetadataBox } + + )} ); -} - -export const DataQueryForm: React.FC<{ - dataLoaderType: string, - availableTables: {name: string, fields: string[]}[], - dataLoaderParams: Record, - onImport: () => void, - onFinish: (status: "success" | "error", message: string) => void -}> = ({dataLoaderType, availableTables, dataLoaderParams, onImport, onFinish}) => { - - let activeModel = useSelector(dfSelectors.getActiveModel); - - const [selectedTables, setSelectedTables] = useState(availableTables.map(t => t.name).slice(0, 5)); - - const [waiting, setWaiting] = useState(false); - - const [query, setQuery] = useState("-- query the data source / describe your goal and ask AI to help you write the query\n"); - const [queryResult, setQueryResult] = useState<{ - status: string, - message: string, - sample: any[], - code: string, - } | undefined>(undefined); - const [queryResultName, setQueryResultName] = useState(""); - - const aiCompleteQuery = (query: string) => { - if (queryResult?.status === "error") { - setQueryResult(undefined); - } - let data = { - data_source_metadata: { - data_loader_type: dataLoaderType, - tables: availableTables.filter(t => selectedTables.includes(t.name)) - }, - query: query, - model: activeModel - } - setWaiting(true); - fetch(getUrls().QUERY_COMPLETION, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(data) - }) - .then(response => response.json()) - .then(data => { - setWaiting(false); - if (data.status === "ok") { - setQuery(data.query); - } else { - onFinish("error", data.reasoning); - } - }) - .catch(error => { - setWaiting(false); - onFinish("error", `Failed to complete query please try again.`); - }); - } - - const handleViewQuerySample = (query: string) => { - setQueryResult(undefined); - setWaiting(true); - fetch(getUrls().DATA_LOADER_VIEW_QUERY_SAMPLE, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - data_loader_type: dataLoaderType, - data_loader_params: dataLoaderParams, - query: query - }) - }) - .then(response => response.json()) - .then(data => { - setWaiting(false); - if (data.status === "success") { - setQueryResult({ - status: "success", - message: "Data loaded successfully", - sample: data.sample, - code: query - }); - let newName = `r_${Math.random().toString(36).substring(2, 4)}`; - setQueryResultName(newName); - } else { - setQueryResult({ - status: "error", - message: data.message, - sample: [], - code: query - }); - } - }) - .catch(error => { - setWaiting(false); - setQueryResult({ - status: "error", - message: `Failed to view query sample, please try again.`, - sample: [], - code: query - }); - }); - } - - const handleImportQueryResult = () => { - setWaiting(true); - fetch(getUrls().DATA_LOADER_INGEST_DATA_FROM_QUERY, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - data_loader_type: dataLoaderType, - data_loader_params: dataLoaderParams, - query: queryResult?.code ?? query, - name_as: queryResultName - }) - }) - .then(response => response.json()) - .then(data => { - setWaiting(false); - if (data.status === "success") { - onFinish("success", "Data imported successfully"); - } else { - onFinish("error", data.reasoning); - } - }) - .catch(error => { - setWaiting(false); - onFinish("error", `Failed to import data, please try again.`); - }); - } - - let queryResultBox = queryResult?.status === "success" && queryResult.sample.length > 0 ? [ - - ({id: t, label: t}))} rowsPerPageNum={-1} compact={false} /> - , - - - setQueryResultName(event.target.value)} - /> - - - ] : []; - - return ( - - {waiting && - - } - - - query from tables: - - {availableTables.map((table) => ( - : undefined} - color={selectedTables.includes(table.name) ? "primary" : "default"} variant="outlined" - sx={{ fontSize: 11, margin: 0.25, - height: 20, borderRadius: 0.5, - borderColor: selectedTables.includes(table.name) ? "primary.main" : "rgba(0, 0, 0, 0.1)", - color: selectedTables.includes(table.name) ? "primary.main" : "text.secondary", - '&:hover': { - backgroundColor: "rgba(0, 0, 0, 0.07)", - } - }} - size="small" - onClick={() => { - setSelectedTables(selectedTables.includes(table.name) ? selectedTables.filter(t => t !== table.name) : [...selectedTables, table.name]); - }} - /> - ))} - - - - { - setQuery(tempCode); - }} - highlight={code => Prism.highlight(code, Prism.languages.sql, 'sql')} - padding={10} - style={{ - minHeight: queryResult ? 60 : 200, - fontFamily: '"Fira code", "Fira Mono", monospace', - fontSize: 12, - paddingBottom: '24px', - backgroundColor: "rgba(0, 0, 0, 0.03)", - overflowY: "auto" - }} - /> - - {queryResult?.status === "error" && - - {queryResult?.message} - - } - - - {queryResult?.status === "error" && } - - - {queryResult && queryResultBox} - - - ) } \ No newline at end of file diff --git a/src/views/DataFormulator.tsx b/src/views/DataFormulator.tsx index ead4e94..c71aaea 100644 --- a/src/views/DataFormulator.tsx +++ b/src/views/DataFormulator.tsx @@ -41,17 +41,14 @@ import { VisualizationViewFC } from './VisualizationView'; import { ConceptShelf } from './ConceptShelf'; import { DndProvider } from 'react-dnd' import { HTML5Backend } from 'react-dnd-html5-backend' -import { TableCopyDialogV2, DatasetSelectionDialog } from './TableSelectionView'; -import { TableUploadDialog } from './TableSelectionView'; import { toolName } from '../app/App'; import { DataThread } from './DataThread'; import dfLogo from '../assets/df-logo.png'; import exampleImageTable from "../assets/example-image-table.png"; import { ModelSelectionButton } from './ModelSelectionDialog'; -import { DBTableSelectionDialog } from './DBTableManager'; import { getUrls } from '../app/utils'; -import { DataLoadingChatDialog } from './DataLoadingChat'; +import { UnifiedDataUploadDialog, UploadTabType } from './UnifiedDataUploadDialog'; import { ReportView } from './ReportView'; import { ExampleSession, exampleSessions, ExampleSessionCard } from './ExampleSessions'; @@ -65,6 +62,15 @@ export const DataFormulatorFC = ({ }) => { const dispatch = useDispatch(); + // State for unified data upload dialog + const [uploadDialogOpen, setUploadDialogOpen] = useState(false); + const [uploadDialogInitialTab, setUploadDialogInitialTab] = useState('menu'); + + const openUploadDialog = (tab: UploadTabType) => { + setUploadDialogInitialTab(tab); + setUploadDialogOpen(true); + }; + const handleLoadExampleSession = (session: ExampleSession) => { dispatch(dfActions.addMessages({ timestamp: Date.now(), @@ -274,19 +280,32 @@ export const DataFormulatorFC = ({ }) => { + maxWidth: 1100, fontSize: 32, color: alpha(theme.palette.text.primary, 0.8), + lineHeight: 2, + '& span': { + textTransform: 'uppercase', + letterSpacing: '0.02em', + textDecoration: 'underline', textUnderlineOffset: '0.2em', + cursor: 'pointer', color: theme.palette.primary.main, + '&:hover': { + color: theme.palette.primary.dark, + } + }}}> To begin, - extract}/>{' '} + {' '} openUploadDialog('extract')}>extract{' '} data from images or text documents, load {' '} - examples}/>, + {' '} openUploadDialog('explore')}>examples{' '}, upload data from{' '} - clipboard} disabled={false}/> or {' '} - files} disabled={false}/>, - + {' '} openUploadDialog('paste')}>clipboard or {' '} + {' '} openUploadDialog('upload')}>files{' '}, or connect to a{' '} - database}/>. + {' '} openUploadDialog('database')}>database{' '}. + setUploadDialogOpen(false)} + initialTab={uploadDialogInitialTab} + /> @@ -331,7 +350,7 @@ export const DataFormulatorFC = ({ }) => { zIndex: 1000, }}> - + {toolName} diff --git a/src/views/DataLoadingChat.tsx b/src/views/DataLoadingChat.tsx index a3273a5..6602149 100644 --- a/src/views/DataLoadingChat.tsx +++ b/src/views/DataLoadingChat.tsx @@ -4,9 +4,7 @@ import * as React from 'react'; import { useEffect, useRef, useState } from 'react'; -import { Box, Button, Divider, IconButton, Typography, Dialog, DialogTitle, DialogContent, Tooltip, CircularProgress } from '@mui/material'; -import RestartAltIcon from '@mui/icons-material/RestartAlt'; -import CloseIcon from '@mui/icons-material/Close'; +import { Box, Button, Divider, IconButton, Typography, Tooltip, CircularProgress, alpha, useTheme } from '@mui/material'; import { useDispatch, useSelector } from 'react-redux'; @@ -32,7 +30,7 @@ const getUniqueTableName = (baseName: string, existingNames: Set): strin }; export const DataLoadingChat: React.FC = () => { - + const theme = useTheme(); const dispatch = useDispatch(); const inputBoxRef = useRef<(() => void) | null>(null); const abortControllerRef = useRef(null); @@ -89,12 +87,6 @@ export const DataLoadingChat: React.FC = () => { } }; - if (!existOutputBlocks && !streamingContent) { - return - - - } - const thinkingBanner = ( { ); - - let chatCard = ( - - - {/* Left: Chat panel */} - + - - {threadsComponent} + gap: 2, + }}> + - + ); + } - - - {streamingContent && ( - - {thinkingBanner} - - {streamingContent.trim()} - - - )} - - {/* Right: Data preview panel */} - {(existOutputBlocks && !streamingContent) && ( - + + {/* Left sidebar - Thread list (similar to DBTablePane) */} + + - - - {selectedTable && ( - - {selectedTable?.name} - - )} - - - - {selectedTable ? ( - + overflowY: 'auto', + overflowX: 'hidden', + flex: 1, + minHeight: 0, + height: '100%', + position: 'relative', + overscrollBehavior: 'contain', + px: 0.5, + pt: 1 + }}> + {threadsComponent.length > 0 ? ( + threadsComponent ) : ( - - No data available + + No extraction threads yet )} + + + + + - {/* Bottom submit bar */} - - - + {selectedTable ? ( + + + + ) : ( + + + Select a table from the left to preview + + + )} + + {/* Bottom submit bar */} + {selectedTable && ( + + + + + )} + - + ) : null} - )} + ); - - return chatCard; -}; - -export interface DataLoadingChatDialogProps { - buttonElement?: any; - disabled?: boolean; - onOpen?: () => void; - // Controlled mode props - open?: boolean; - onClose?: () => void; -} - -export const DataLoadingChatDialog: React.FC = ({ - buttonElement, - disabled = false, - onOpen, - open: controlledOpen, - onClose, -}) => { - const [internalOpen, setInternalOpen] = useState(false); - const dispatch = useDispatch(); - const dataCleanBlocks = useSelector((state: DataFormulatorState) => state.dataCleanBlocks); - - // Support both controlled and uncontrolled modes - const isControlled = controlledOpen !== undefined; - const dialogOpen = isControlled ? controlledOpen : internalOpen; - const setDialogOpen = isControlled - ? (open: boolean) => { if (!open && onClose) onClose(); } - : setInternalOpen; - - return ( - <> - {buttonElement && ( - - )} - setDialogOpen(false)} - open={dialogOpen} - sx={{ '& .MuiDialog-paper': { maxWidth: '100%', maxHeight: 840, minWidth: 800 } }} - > - - Extract Data - {dataCleanBlocks.length > 0 && - { - dispatch(dfActions.resetDataCleanBlocks()); - }}> - - - } - setDialogOpen(false)} - aria-label="close" - > - - - - - - - - - ); }; - diff --git a/src/views/DataThread.tsx b/src/views/DataThread.tsx index d873edb..d210fee 100644 --- a/src/views/DataThread.tsx +++ b/src/views/DataThread.tsx @@ -22,7 +22,9 @@ import { Popper, Paper, ClickAwayListener, - Badge + Badge, + Menu, + MenuItem, } from '@mui/material'; import { VegaLite } from 'react-vega' @@ -61,10 +63,15 @@ import ChevronLeftIcon from '@mui/icons-material/ChevronLeft'; import ChevronRightIcon from '@mui/icons-material/ChevronRight'; import CloudQueueIcon from '@mui/icons-material/CloudQueue'; import AttachFileIcon from '@mui/icons-material/AttachFile'; +import MoreVertIcon from '@mui/icons-material/MoreVert'; +import RefreshIcon from '@mui/icons-material/Refresh'; import { alpha } from '@mui/material/styles'; import { dfSelectors } from '../app/dfSlice'; +import { RefreshDataDialog } from './RefreshDataDialog'; +import { getUrls } from '../app/utils'; +import { AppDispatch } from '../app/store'; export const ThinkingBanner = (message: string, sx?: SxProps) => ( = function ({ + scrollRef, + leafTables, + chartElements, + sx +}) { + const theme = useTheme(); + + return ( + + + + + workspace + + + + + + + + ); +} + let SingleThreadGroupView: FC<{ scrollRef: any, threadIdx: number, leafTables: DictTable[]; chartElements: { tableId: string, chartId: string, element: any }[]; usedIntermediateTableIds: string[], + compact?: boolean, // When true, only show table cards in a simple column (for thread0) sx?: SxProps }> = function ({ scrollRef, @@ -471,6 +528,7 @@ let SingleThreadGroupView: FC<{ leafTables, chartElements, usedIntermediateTableIds, // tables that have been used + compact = false, sx }) { @@ -490,6 +548,14 @@ let SingleThreadGroupView: FC<{ const [selectedTableForMetadata, setSelectedTableForMetadata] = useState(null); const [metadataAnchorEl, setMetadataAnchorEl] = useState(null); + // Table menu state + const [tableMenuAnchorEl, setTableMenuAnchorEl] = useState(null); + const [selectedTableForMenu, setSelectedTableForMenu] = useState(null); + + // Refresh data dialog state + const [refreshDialogOpen, setRefreshDialogOpen] = useState(false); + const [selectedTableForRefresh, setSelectedTableForRefresh] = useState(null); + const [isRefreshing, setIsRefreshing] = useState(false); let handleUpdateTableDisplayId = (tableId: string, displayId: string) => { dispatch(dfActions.updateTableDisplayId({ @@ -519,6 +585,127 @@ let SingleThreadGroupView: FC<{ } }; + // Table menu handlers + const handleOpenTableMenu = (table: DictTable, anchorEl: HTMLElement) => { + setSelectedTableForMenu(table); + setTableMenuAnchorEl(anchorEl); + }; + + const handleCloseTableMenu = () => { + setTableMenuAnchorEl(null); + setSelectedTableForMenu(null); + }; + + // Refresh data handlers + const handleOpenRefreshDialog = (table: DictTable) => { + setSelectedTableForRefresh(table); + setRefreshDialogOpen(true); + handleCloseTableMenu(); + }; + + const handleCloseRefreshDialog = () => { + setRefreshDialogOpen(false); + setSelectedTableForRefresh(null); + }; + + // Function to refresh derived tables + const refreshDerivedTables = async (sourceTableId: string, newRows: any[]) => { + // Find all tables that are derived from this source table + const derivedTables = tables.filter(t => t.derive?.source?.includes(sourceTableId)); + + for (const derivedTable of derivedTables) { + if (derivedTable.derive && derivedTable.derive.code) { + // Gather all parent tables for this derived table + const parentTableData = derivedTable.derive.source.map(sourceId => { + const sourceTable = tables.find(t => t.id === sourceId); + if (sourceTable) { + // Use the new rows if this is the table being refreshed + const rows = sourceId === sourceTableId ? newRows : sourceTable.rows; + return { + name: sourceTable.id, + rows: rows + }; + } + return null; + }).filter(t => t !== null); + + if (parentTableData.length > 0) { + try { + const response = await fetch(getUrls().REFRESH_DERIVED_DATA, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + input_tables: parentTableData, + code: derivedTable.derive.code + }) + }); + + const result = await response.json(); + if (result.status === 'ok' && result.rows) { + // Update the derived table with new rows + dispatch(dfActions.updateTableRows({ + tableId: derivedTable.id, + rows: result.rows + })); + + // Recursively refresh tables derived from this one + await refreshDerivedTables(derivedTable.id, result.rows); + } else { + console.error(`Failed to refresh derived table ${derivedTable.id}:`, result.message); + dispatch(dfActions.addMessages({ + timestamp: Date.now(), + type: 'error', + component: 'data refresh', + value: `Failed to refresh derived table "${derivedTable.displayId || derivedTable.id}": ${result.message || 'Unknown error'}` + })); + } + } catch (error) { + console.error(`Error refreshing derived table ${derivedTable.id}:`, error); + dispatch(dfActions.addMessages({ + timestamp: Date.now(), + type: 'error', + component: 'data refresh', + value: `Error refreshing derived table "${derivedTable.displayId || derivedTable.id}"` + })); + } + } + } + } + }; + + const handleRefreshComplete = async (newRows: any[]) => { + if (!selectedTableForRefresh) return; + + setIsRefreshing(true); + try { + // Update the source table with new rows + dispatch(dfActions.updateTableRows({ + tableId: selectedTableForRefresh.id, + rows: newRows + })); + + // Refresh all derived tables + await refreshDerivedTables(selectedTableForRefresh.id, newRows); + + dispatch(dfActions.addMessages({ + timestamp: Date.now(), + type: 'success', + component: 'data refresh', + value: `Successfully refreshed data for "${selectedTableForRefresh.displayId || selectedTableForRefresh.id}" and updated derived tables.` + })); + } catch (error) { + console.error('Error during refresh:', error); + dispatch(dfActions.addMessages({ + timestamp: Date.now(), + type: 'error', + component: 'data refresh', + value: `Error refreshing data: ${error}` + })); + } finally { + setIsRefreshing(false); + } + }; + let buildTriggerCard = (trigger: Trigger) => { let selectedClassName = trigger.chart?.id == focusedChartId ? 'selected-card' : ''; @@ -543,7 +730,7 @@ let SingleThreadGroupView: FC<{ ; } - let buildTableCard = (tableId: string) => { + let buildTableCard = (tableId: string, compact = false) => { if (parentTable && tableId == parentTable.id && parentTable.anchored && tableIdList.length > 1) { let table = tables.find(t => t.id == tableId); @@ -673,43 +860,8 @@ let SingleThreadGroupView: FC<{ - {table?.derive == undefined && - { - event.stopPropagation(); - handleOpenMetadataPopup(table!, event.currentTarget); - }} - > - - - } - - {tableDeleteEnabled && - { - event.stopPropagation(); - dispatch(dfActions.deleteTable(tableId)); - }} - > - - - } - - + + {/* For non-derived, non-virtual tables: show dropdown menu with metadata, refresh, delete */} + {table?.derive == undefined && !table?.virtual && ( + + { + event.stopPropagation(); + handleOpenTableMenu(table!, event.currentTarget); + }} + > + + + + )} + + {/* For derived tables or virtual tables: show individual buttons */} + {(table?.derive != undefined || table?.virtual) && ( + <> + {tableDeleteEnabled && + { + event.stopPropagation(); + dispatch(dfActions.deleteTable(tableId)); + }} + > + + + } + + )} @@ -756,7 +950,7 @@ let SingleThreadGroupView: FC<{ backgroundSize: '1px 6px, 3px 100%' }}> } - + {releventChartElements} {agentActionBox} @@ -829,6 +1023,94 @@ let SingleThreadGroupView: FC<{ ; }); + // Compact mode: just show leaf table cards in a simple column + if (compact) { + // For compact mode, ensure highlightedTableIds includes focused table if it's a leaf + if (focusedTableId && leafTableIds.includes(focusedTableId)) { + highlightedTableIds = [focusedTableId]; + } + + return ( + + {leafTables.map((table) => { + const tableCardResult = buildTableCard(table.id, compact); + // buildTableCard returns an array [regularTableBox, chartBox] + // In compact mode, we want to show them stacked + return ( + + {tableCardResult} + + ); + })} + + e.stopPropagation()} + > + { + e.stopPropagation(); + if (selectedTableForMenu) { + handleOpenMetadataPopup(selectedTableForMenu, tableMenuAnchorEl!); + } + handleCloseTableMenu(); + }} + sx={{ fontSize: '12px', display: 'flex', alignItems: 'center', gap: 1 }} + > + + {selectedTableForMenu?.attachedMetadata ? "Edit metadata" : "Attach metadata"} + + { + e.stopPropagation(); + if (selectedTableForMenu) { + handleOpenRefreshDialog(selectedTableForMenu); + } + }} + sx={{ fontSize: '12px', display: 'flex', alignItems: 'center', gap: 1 }} + > + + Refresh data + + { + e.stopPropagation(); + if (selectedTableForMenu) { + dispatch(dfActions.deleteTable(selectedTableForMenu.id)); + } + handleCloseTableMenu(); + }} + disabled={selectedTableForMenu ? tables.some(t => t.derive?.trigger.tableId === selectedTableForMenu.id) : true} + sx={{ fontSize: '12px', display: 'flex', alignItems: 'center', gap: 1, color: 'warning.main' }} + > + + Delete table + + + {selectedTableForRefresh && ( + + )} + + ); + } + return - {`thread - ${threadIdx + 1}`} + {threadIdx === -1 ? 'thread0' : `thread - ${threadIdx + 1}`} @@ -888,6 +1170,67 @@ let SingleThreadGroupView: FC<{ initialValue={selectedTableForMetadata?.attachedMetadata || ''} tableName={selectedTableForMetadata?.displayId || selectedTableForMetadata?.id || ''} /> + + {/* Table actions menu for non-derived, non-virtual tables */} + e.stopPropagation()} + > + { + e.stopPropagation(); + if (selectedTableForMenu) { + handleOpenMetadataPopup(selectedTableForMenu, tableMenuAnchorEl!); + } + handleCloseTableMenu(); + }} + sx={{ fontSize: '12px', display: 'flex', alignItems: 'center', gap: 1 }} + > + + {selectedTableForMenu?.attachedMetadata ? "Edit metadata" : "Attach metadata"} + + { + e.stopPropagation(); + if (selectedTableForMenu) { + handleOpenRefreshDialog(selectedTableForMenu); + } + }} + sx={{ fontSize: '12px', display: 'flex', alignItems: 'center', gap: 1 }} + > + + Refresh data + + { + e.stopPropagation(); + if (selectedTableForMenu) { + dispatch(dfActions.deleteTable(selectedTableForMenu.id)); + } + handleCloseTableMenu(); + }} + disabled={selectedTableForMenu ? tables.some(t => t.derive?.trigger.tableId === selectedTableForMenu.id) : true} + sx={{ fontSize: '12px', display: 'flex', alignItems: 'center', gap: 1, color: 'warning.main' }} + > + + Delete table + + + + {/* Refresh data dialog */} + {selectedTableForRefresh && ( + + )} } @@ -1179,7 +1522,22 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { return aOrders.length - bOrders.length; }); - let leafTableGroups = leafTables.reduce((groups: { [groupId: string]: DictTable[] }, leafTable) => { + // Identify hanging tables (tables with no descendants or parents) + let isHangingTable = (table: DictTable) => { + // A table is hanging if: + // 1. It has no derive.source (no parent) + // 2. No other table derives from it (no descendants) + const hasNoParent = table.derive == undefined; + const hasNoDescendants = !tables.some(t => t.derive?.trigger.tableId == table.id); + return hasNoParent && hasNoDescendants; + }; + + // Separate hanging tables from regular leaf tables + let hangingTables = leafTables.filter(t => isHangingTable(t)); + let regularLeafTables = leafTables.filter(t => !isHangingTable(t)); + + // Build groups for regular leaf tables (excluding hanging tables) + let leafTableGroups = regularLeafTables.reduce((groups: { [groupId: string]: DictTable[] }, leafTable) => { // Get the immediate parent table ID (first trigger in the chain) const triggers = getTriggers(leafTable, tables); const immediateParentTableId = triggers.length > 0 ? triggers[triggers.length - 1].tableId : 'root'; @@ -1203,8 +1561,38 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { return groups; }, {}); - let drawerOpen = threadDrawerOpen && leafTables.length > 1; - let collaposedViewWidth = Math.max(...Object.values(leafTableGroups).map(x => x.length)) > 1 ? 248 : 232 + // Filter threads to only include those with length > 1 + let filteredLeafTableGroups: { [groupId: string]: DictTable[] } = {}; + Object.entries(leafTableGroups).forEach(([groupId, groupTables]) => { + // Calculate thread length: count all tables in the thread chain + const threadLength = groupTables.reduce((maxLength, leafTable) => { + const triggers = getTriggers(leafTable, tables); + // Thread length = number of triggers + 1 (the leaf table itself) + return Math.max(maxLength, triggers.length + 1); + }, 0); + + // Only include threads with length > 1 + if (threadLength > 1) { + filteredLeafTableGroups[groupId] = groupTables; + } else { + // Add single-table threads to hanging tables (they go to thread0) + groupTables.forEach(table => { + if (!hangingTables.includes(table)) { + hangingTables.push(table); + } + }); + } + }); + + // Create thread0 group for hanging tables + let thread0Group: { [groupId: string]: DictTable[] } = {}; + if (hangingTables.length > 0) { + thread0Group['thread0'] = hangingTables; + } + + let drawerOpen = threadDrawerOpen && (Object.keys(filteredLeafTableGroups).length > 0 || hangingTables.length > 0); + let allGroupsForWidth = { ...filteredLeafTableGroups, ...thread0Group }; + let collaposedViewWidth = Math.max(...Object.values(allGroupsForWidth).map(x => x.length)) > 1 ? 248 : 232 let view = = function ({ sx }) { p: 1, transition: 'max-width 0.1s linear', // Smooth width transition }}> - {Object.entries(leafTableGroups).map(([groupId, leafTables], i) => { - - let usedIntermediateTableIds = Object.values(leafTableGroups).slice(0, i).flat() + {/* Render thread0 (hanging tables) first if it exists - using compact view */} + {Object.entries(thread0Group).map(([groupId, leafTables], i) => { + return 1 ? '216px' : '200px', + transition: 'all 0.3s ease', + }} /> + })} + {/* Render regular threads (length > 1) */} + {Object.entries(filteredLeafTableGroups).map(([groupId, leafTables], i) => { + // Calculate used tables from thread0 and previous threads + let usedIntermediateTableIds = Object.values(thread0Group).flat() .map(x => [ ...getTriggers(x, tables).map(y => y.tableId) || []]).flat(); - let usedLeafTableIds = Object.values(leafTableGroups).slice(0, i).flat().map(x => x.id); + let usedLeafTableIds = Object.values(thread0Group).flat().map(x => x.id); + + // Add tables from previous regular threads + const previousThreadGroups = Object.values(filteredLeafTableGroups).slice(0, i); + usedIntermediateTableIds = [...usedIntermediateTableIds, ...previousThreadGroups.flat() + .map(x => [ ...getTriggers(x, tables).map(y => y.tableId) || []]).flat()]; + usedLeafTableIds = [...usedLeafTableIds, ...previousThreadGroups.flat().map(x => x.id)]; return = function ({ sx }) { })} + // Calculate total thread count (thread0 + regular threads) + let totalThreadCount = Object.keys(filteredLeafTableGroups).length + (Object.keys(thread0Group).length > 0 ? 1 : 0); + let threadIndices: number[] = []; + if (Object.keys(thread0Group).length > 0) { + threadIndices.push(-1); // thread0 + } + threadIndices.push(...Array.from({length: Object.keys(filteredLeafTableGroups).length}, (_, i) => i)); + let jumpButtonsDrawerOpen = - {_.chunk(Array.from({length: Object.keys(leafTableGroups).length}, (_, i) => i), 3).map((group, groupIdx) => { - const startNum = group[0] + 1; - const endNum = group[group.length - 1] + 1; - const label = startNum === endNum ? `${startNum}` : `${startNum}-${endNum}`; + {_.chunk(threadIndices, 3).map((group, groupIdx) => { + const getLabel = (idx: number) => idx === -1 ? '0' : String(idx + 1); + const startNum = getLabel(group[0]); + const endNum = getLabel(group[group.length - 1]); + const label = startNum === endNum ? startNum : `${startNum}-${endNum}`; return ( @@ -1265,8 +1689,9 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { const currentIndex = Array.from(document.querySelectorAll('[data-thread-index]')).reduce((closest, element) => { const rect = element.getBoundingClientRect(); const distance = Math.abs(rect.left + rect.width/2 - viewportCenter); + const idx = parseInt(element.getAttribute('data-thread-index') || '0'); if (!closest || distance < closest.distance) { - return { index: parseInt(element.getAttribute('data-thread-index') || '0'), distance }; + return { index: idx, distance }; } return closest; }, null as { index: number, distance: number } | null)?.index || 0; @@ -1294,21 +1719,24 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { let jumpButtonDrawerClosed = - {Object.keys(leafTableGroups).map((groupId, idx) => ( - - { - const threadElement = document.querySelector(`[data-thread-index="${idx}"]`); - threadElement?.scrollIntoView({ behavior: 'smooth' }); - }} - > - {idx + 1} - - - ))} + {threadIndices.map((threadIdx) => { + const label = threadIdx === -1 ? '0' : String(threadIdx + 1); + return ( + + { + const threadElement = document.querySelector(`[data-thread-index="${threadIdx}"]`); + threadElement?.scrollIntoView({ behavior: 'smooth' }); + }} + > + {label} + + + ); + })} let jumpButtons = drawerOpen ? jumpButtonsDrawerOpen : jumpButtonDrawerClosed; @@ -1338,7 +1766,7 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { { + disabled={totalThreadCount <= 1} onClick={() => { setThreadDrawerOpen(true); }}> diff --git a/src/views/DerivedDataDialog.tsx b/src/views/DerivedDataDialog.tsx deleted file mode 100644 index aeb7785..0000000 --- a/src/views/DerivedDataDialog.tsx +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { FC } from 'react' -import { - Card, - Box, - Typography, - Dialog, - DialogTitle, - DialogContent, - DialogActions, - Button, - Radio, - styled, - FormControlLabel, - CardContent, - ButtonGroup, -} from '@mui/material'; - -import React from 'react'; - -import { assembleVegaChart } from '../app/utils'; -import { Chart } from '../components/ComponentType'; -import { useSelector } from 'react-redux'; -import { DataFormulatorState } from '../app/dfSlice'; - -import { createDictTable, DictTable } from '../components/ComponentType'; -import { CodeBox } from './VisualizationView'; -import embed from 'vega-embed'; -import { CustomReactTable } from './ReactTable'; - -import DeleteIcon from '@mui/icons-material/Delete'; -import SaveIcon from '@mui/icons-material/Save'; - -export interface DerivedDataDialogProps { - chart: Chart, - candidateTables: DictTable[], - open: boolean, - handleCloseDialog: () => void, - handleSelection: (selectIndex: number) => void, - handleDeleteChart: () => void, - bodyOnly?: boolean, -} - -export const DerivedDataDialog: FC = function DerivedDataDialog({ - chart, candidateTables, open, handleCloseDialog, handleSelection, handleDeleteChart, bodyOnly }) { - - let direction = candidateTables.length > 1 ? "horizontal" : "horizontal" ; - - let [selectionIdx, setSelectionIdx] = React.useState(0); - const conceptShelfItems = useSelector((state: DataFormulatorState) => state.conceptShelfItems); - - let body = - - - {candidateTables.map((table, idx) => { - let code = table.derive?.code || ""; - let extTable = structuredClone(table.rows); - - let assembledChart: any = assembleVegaChart(chart.chartType, chart.encodingMap, conceptShelfItems, extTable, table.metadata); - assembledChart["background"] = "transparent"; - // chart["autosize"] = { - // "type": "fit", - // "contains": "padding" - // }; - const id = `chart-dialog-element-${idx}`; - - const element = - setSelectionIdx(idx)}> - ; - - embed('#' + id, assembledChart, { actions: false, renderer: "canvas" }).then(function (result) { - // Access the Vega view instance (https://vega.github.io/vega/docs/api/view/) as result.view - if (result.view.container()?.getElementsByTagName("canvas")) { - let comp = result.view.container()?.getElementsByTagName("canvas")[0]; - - // Doesn't seem like width & height are actual numbers here on Edge bug - // let width = parseInt(comp?.style.width as string); - // let height = parseInt(comp?.style.height as string); - if (comp) { - const { width, height } = comp.getBoundingClientRect(); - //console.log(`THUMB: width = ${width} height = ${height}`); - if (width > 240 || height > 180) { - let ratio = width / height; - let fixedWidth = width; - if (ratio * 180 < width) { - fixedWidth = ratio * 180; - } - if (fixedWidth > 240) { - fixedWidth = 240; - } - comp?.setAttribute("style", `max-width: 240px; max-height: 180px; width: ${Math.round(fixedWidth)}px; height: ${Math.round(fixedWidth / ratio)}px; `); - } - - } else { - console.log("THUMB: Could not get Canvas HTML5 element") - } - } - }).catch((reason) => { - // console.log(reason) - // console.error(reason) - }); - - let simpleTableView = (t: DictTable) => { - let colDefs = t.names.map(name => { - return { - id: name, label: name, minWidth: 30, align: undefined, - format: (value: any) => `${value}`, source: conceptShelfItems.find(f => f.name == name)?.source - } - }) - return - - - } - - return {setSelectionIdx(idx)}} - sx={{minWidth: "280px", maxWidth: "1920px", display: "flex", flexGrow: 1, margin: "6px", - border: selectionIdx == idx ? "2px solid rgb(2 136 209 / 0.7)": "1px solid rgba(33, 33, 33, 0.1)"}}> - - } - label={{`candidate-${idx+1} (${candidateTables[idx].id})`}} /> - - - {element} - - - - {simpleTableView(createDictTable(table.id, extTable))} - - - - - - - - - })} - - - - if (bodyOnly) { - return - - Transformation from {candidateTables[0].derive?.source} - - {body} - - - - {/* */} - - - - - - ; - } - - return ( - - Derived Data Candidates - - {body} - - - - - - - ); -} \ No newline at end of file diff --git a/src/views/EncodingShelfCard.tsx b/src/views/EncodingShelfCard.tsx index d66e357..bb230df 100644 --- a/src/views/EncodingShelfCard.tsx +++ b/src/views/EncodingShelfCard.tsx @@ -250,111 +250,6 @@ export const TriggerCard: FC<{ } -// Add this component before EncodingShelfCard -const UserActionTableSelector: FC<{ - requiredActionTableIds: string[], - userSelectedActionTableIds: string[], - tables: DictTable[], - updateUserSelectedActionTableIds: (tableIds: string[]) => void, - requiredTableIds?: string[] -}> = ({ requiredActionTableIds, userSelectedActionTableIds, tables, updateUserSelectedActionTableIds, requiredTableIds = [] }) => { - const [anchorEl, setAnchorEl] = useState(null); - const open = Boolean(anchorEl); - - let actionTableIds = [...requiredActionTableIds, ...userSelectedActionTableIds.filter(id => !requiredActionTableIds.includes(id))]; - - const handleClick = (event: React.MouseEvent) => { - setAnchorEl(event.currentTarget); - }; - - const handleClose = () => { - setAnchorEl(null); - }; - - const handleTableSelect = (table: DictTable) => { - if (!actionTableIds.includes(table.id)) { - updateUserSelectedActionTableIds([...userSelectedActionTableIds, table.id]); - } - handleClose(); - }; - - return ( - - {actionTableIds.map((tableId) => { - const isRequired = requiredTableIds.includes(tableId); - return ( - t.id == tableId)?.displayId} - size="small" - sx={{ - height: 16, - fontSize: '10px', - borderRadius: '0px', - bgcolor: isRequired ? 'rgba(25, 118, 210, 0.2)' : 'rgba(25, 118, 210, 0.1)', // darker blue for required - color: 'rgba(0, 0, 0, 0.7)', - '& .MuiChip-label': { - pl: '4px', - pr: '6px' - } - }} - deleteIcon={} - onDelete={isRequired ? undefined : () => updateUserSelectedActionTableIds(actionTableIds.filter(id => id !== tableId))} - /> - ); - })} - - - - - - - - - {tables - .map((table) => { - const isSelected = !!actionTableIds.find(t => t === table.id); - return ( - handleTableSelect(table)} - sx={{ - fontSize: '12px', - display: 'flex', - justifyContent: 'space-between', - alignItems: 'center' - }} - > - {table.displayId} - - ); - }) - } - - - ); -}; - export const EncodingShelfCard: FC = function ({ chartId }) { const theme = useTheme(); @@ -363,7 +258,6 @@ export const EncodingShelfCard: FC = function ({ chartId const tables = useSelector((state: DataFormulatorState) => state.tables); const config = useSelector((state: DataFormulatorState) => state.config); const agentRules = useSelector((state: DataFormulatorState) => state.agentRules); - let existMultiplePossibleBaseTables = tables.filter(t => t.derive == undefined || t.anchored).length > 1; let activeModel = useSelector(dfSelectors.getActiveModel); let allCharts = useSelector(dfSelectors.getAllCharts); @@ -401,9 +295,7 @@ export const EncodingShelfCard: FC = function ({ chartId // Check if chart is available let isChartAvailable = checkChartAvailability(chart, conceptShelfItems, currentTable.rows); - // Add this state - const [userSelectedActionTableIds, setUserSelectedActionTableIds] = useState([]); - + // Consolidated chart state - maps chartId to its ideas, thinkingBuffer, and loading state const [chartState, setChartState] = useState = function ({ chartId // Add state for developer message dialog const [devMessageOpen, setDevMessageOpen] = useState(false); - - // Update the handler to use state - const handleUserSelectedActionTableChange = (newTableIds: string[]) => { - setUserSelectedActionTableIds(newTableIds); - }; + let encodingBoxGroups = Object.entries(ChannelGroups) .filter(([group, channelList]) => channelList.some(ch => Object.keys(encodingMap).includes(ch))) @@ -479,7 +367,7 @@ export const EncodingShelfCard: FC = function ({ chartId let requiredActionTables = selectBaseTables(activeFields, currentTable, tables); let actionTableIds = [ ...requiredActionTables.map(t => t.id), - ...userSelectedActionTableIds.filter(id => !requiredActionTables.map(t => t.id).includes(id)) + ...tables.filter(t => t.derive === undefined || t.anchored).map(t => t.id).filter(id => !requiredActionTables.map(t => t.id).includes(id)) ]; let getIdeasForVisualization = async () => { @@ -1155,13 +1043,6 @@ export const EncodingShelfCard: FC = function ({ chartId let channelComponent = ( - {existMultiplePossibleBaseTables && t.id)} - userSelectedActionTableIds={userSelectedActionTableIds} - tables={tables.filter(t => t.derive === undefined || t.anchored)} - updateUserSelectedActionTableIds={handleUserSelectedActionTableChange} - requiredTableIds={requiredActionTables.map(t => t.id)} - />} + fileInputRef.current?.click()} + > + + + Drag & drop file here + + + or Browse + + + Supported: CSV, TSV, JSON, Excel (xlsx, xls) + + + + )} + + + + setUrlContent(e.target.value.trim())} + disabled={isLoading} + error={urlContent !== '' && !hasValidUrlSuffix} + helperText={urlContent !== '' && !hasValidUrlSuffix ? 'URL should link to a .csv, .tsv, or .json file' : ''} + size="small" + sx={{ + '& .MuiInputBase-input': { + fontSize: '0.875rem', + }, + '& .MuiInputBase-input::placeholder': { + fontSize: '0.875rem', + }, + '& .MuiFormHelperText-root': { + fontSize: '0.75rem', + }, + }} + /> + + + + + {tabValue === 0 && ( + + )} + {tabValue === 2 && ( + + )} + + + ); +}; diff --git a/src/views/TableSelectionView.tsx b/src/views/TableSelectionView.tsx index 4245025..ba3cbba 100644 --- a/src/views/TableSelectionView.tsx +++ b/src/views/TableSelectionView.tsx @@ -2,63 +2,13 @@ // Licensed under the MIT License. import * as React from 'react'; -import validator from 'validator'; -import DOMPurify from 'dompurify'; +import { useEffect } from 'react'; -import Tabs from '@mui/material/Tabs'; -import Tab from '@mui/material/Tab'; import Typography from '@mui/material/Typography'; import Box from '@mui/material/Box'; -import { alpha, Button, Collapse, Dialog, DialogActions, DialogContent, DialogTitle, Divider, - IconButton, Input, CircularProgress, LinearProgress, Paper, TextField, useTheme, - Card, Tooltip, Link} from '@mui/material'; +import { Button, Card, Paper } from '@mui/material'; import { CustomReactTable } from './ReactTable'; -import { DictTable } from "../components/ComponentType"; - -import DeleteIcon from '@mui/icons-material/Delete'; -import { getUrls } from '../app/utils'; -import { createTableFromFromObjectArray, createTableFromText, loadTextDataWrapper, loadBinaryDataWrapper } from '../data/utils'; - -import CloseIcon from '@mui/icons-material/Close'; - -import { DataFormulatorState, dfActions, dfSelectors, fetchFieldSemanticType } from '../app/dfSlice'; -import { useDispatch, useSelector } from 'react-redux'; -import { useEffect, useState, useCallback } from 'react'; -import { AppDispatch } from '../app/store'; - -interface TabPanelProps { - children?: React.ReactNode; - index: number; - value: number; -} - -function TabPanel(props: TabPanelProps) { - const { children, value, index, ...other } = props; - - return ( - - ); -} - -function a11yProps(index: number) { - return { - id: `vertical-tab-${index}`, - 'aria-controls': `vertical-tabpanel-${index}`, - }; -} +import { createTableFromFromObjectArray } from '../data/utils'; // Update the interface to support multiple tables per dataset export interface DatasetMetadata { @@ -106,704 +56,113 @@ export const DatasetSelectionView: React.FC = functio } return ( - + {/* Button navigation */} - {datasetTitles.map((title, i) => ( - - ))} + + {datasetTitles.map((title, i) => ( + + ))} + {/* Content area */} - - {datasets.map((dataset, i) => { - if (dataset.name !== selectedDatasetName) return null; - - let tableComponents = dataset.tables.map((table, j) => { - let t = createTableFromFromObjectArray(table.table_name, table.sample, true); - let maxDisplayRows = dataset.tables.length > 1 ? 5 : 9; - if (t.rows.length < maxDisplayRows) { - maxDisplayRows = t.rows.length - 1; - } - let sampleRows = [ - ...t.rows.slice(0,maxDisplayRows), - Object.fromEntries(t.names.map(n => [n, "..."])) - ]; - let colDefs = t.names.map(name => { return { - id: name, label: name, minWidth: 60, align: undefined, format: (v: any) => v, - }}) - - let content = - - - + + + {datasets.map((dataset, i) => { + if (dataset.name !== selectedDatasetName) return null; + + let tableComponents = dataset.tables.map((table, j) => { + let t = createTableFromFromObjectArray(table.table_name, table.sample, true); + let maxDisplayRows = dataset.tables.length > 1 ? 5 : 9; + if (t.rows.length < maxDisplayRows) { + maxDisplayRows = t.rows.length - 1; + } + let sampleRows = [ + ...t.rows.slice(0,maxDisplayRows), + Object.fromEntries(t.names.map(n => [n, "..."])) + ]; + let colDefs = t.names.map(name => { return { + id: name, label: name, minWidth: 60, align: undefined, format: (v: any) => v, + }}) + + return ( + + + {table.url.split("/").pop()?.split(".")[0]} ({Object.keys(t.rows[0]).length} columns{hideRowNum ? "" : ` ⨉ ${t.rows.length} rows`}) + + + + + + + + ) + }); + return ( - - - {table.url.split("/").pop()?.split(".")[0]} ({Object.keys(t.rows[0]).length} columns{hideRowNum ? "" : ` ⨉ ${t.rows.length} rows`}) - - {content} - - ) - }); - - return ( - - - - {dataset.description} [from {dataset.source}] - - - + + + + {dataset.description} [from {dataset.source}] + + + + + {tableComponents} - {tableComponents} - - ); - })} + ); + })} + ); } - -export const DatasetSelectionDialog: React.FC<{ buttonElement: any }> = function DatasetSelectionDialog({ buttonElement }) { - - const [datasetPreviews, setDatasetPreviews] = React.useState([]); - const [tableDialogOpen, setTableDialogOpen] = useState(false); - - React.useEffect(() => { - // Show a loading animation/message while loading - fetch(`${getUrls().EXAMPLE_DATASETS}`) - .then((response) => response.json()) - .then((result) => { - let datasets : DatasetMetadata[] = result.map((info: any) => { - let tables = info["tables"].map((table: any) => { - - if (table["format"] == "json") { - return { - table_name: table["name"], - url: table["url"], - format: table["format"], - sample: table["sample"], - } - } - else if (table["format"] == "csv" || table["format"] == "tsv") { - const delimiter = table["format"] === "csv" ? "," : "\t"; - const rows = table["sample"] - .split("\n") - .map((row: string) => row.split(delimiter)); - - // Treat first row as headers and convert to object array - if (rows.length > 0) { - const headers = rows[0]; - const dataRows = rows.slice(1); - const sampleData = dataRows.map((row: string[]) => { - const obj: any = {}; - headers.forEach((header: string, index: number) => { - obj[header] = row[index] || ''; - }); - return obj; - }); - - return { - table_name: table["name"], - url: table["url"], - format: table["format"], - sample: sampleData, - }; - } - - return { - table_name: table["name"], - url: table["url"], - format: table["format"], - sample: [], - }; - } - }) - return {tables: tables, name: info["name"], description: info["description"], source: info["source"]} - }).filter((t : DatasetMetadata | undefined) => t != undefined); - setDatasetPreviews(datasets); - }); - }, []); - - let dispatch = useDispatch(); - - return <> - - {setTableDialogOpen(false)}} - open={tableDialogOpen} - sx={{ '& .MuiDialog-paper': { maxWidth: '100%', maxHeight: 840, minWidth: 800 } }} - > - Explore - {setTableDialogOpen(false)}} - aria-label="close" - > - - - - - { - setTableDialogOpen(false); - for (let table of dataset.tables) { - fetch(table.url) - .then(res => res.text()) - .then(textData => { - let tableName = table.url.split("/").pop()?.split(".")[0] || 'table-' + Date.now().toString().substring(0, 8); - let dictTable; - if (table.format == "csv") { - dictTable = createTableFromText(tableName, textData); - } else if (table.format == "json") { - dictTable = createTableFromFromObjectArray(tableName, JSON.parse(textData), true); - } - if (dictTable) { - dispatch(dfActions.loadTable(dictTable)); - dispatch(fetchFieldSemanticType(dictTable)); - } - - }); - } - }}/> - - - -} - -export interface TableUploadDialogProps { - buttonElement?: any; - disabled?: boolean; - onOpen?: () => void; - // For external control of file input - fileInputRef?: React.RefObject; -} - -const getUniqueTableName = (baseName: string, existingNames: Set): string => { - let uniqueName = baseName; - let counter = 1; - while (existingNames.has(uniqueName)) { - uniqueName = `${baseName}_${counter}`; - counter++; - } - return uniqueName; -}; - -export const TableUploadDialog: React.FC = ({ buttonElement, disabled, onOpen, fileInputRef }) => { - const dispatch = useDispatch(); - const internalRef = React.useRef(null); - const inputRef = fileInputRef || internalRef; - const existingTables = useSelector((state: DataFormulatorState) => state.tables); - const existingNames = new Set(existingTables.map(t => t.id)); - const serverConfig = useSelector((state: DataFormulatorState) => state.serverConfig); - - let handleFileUpload = (event: React.ChangeEvent): void => { - const files = event.target.files; - - if (files) { - for (let file of files) { - const uniqueName = getUniqueTableName(file.name, existingNames); - - // Check if file is a text type (csv, tsv, json) - if (file.type === 'text/csv' || - file.type === 'text/tab-separated-values' || - file.type === 'application/json' || - file.name.endsWith('.csv') || - file.name.endsWith('.tsv') || - file.name.endsWith('.json')) { - - // Check if file is larger than 5MB - const MAX_FILE_SIZE = 5 * 1024 * 1024; // 5MB in bytes - if (file.size > MAX_FILE_SIZE) { - dispatch(dfActions.addMessages({ - "timestamp": Date.now(), - "type": "error", - "component": "data loader", - "value": `File ${file.name} is too large (${(file.size / (1024 * 1024)).toFixed(2)}MB), upload it via DATABASE option instead.` - })); - continue; // Skip this file and process the next one - } - - // Handle text files - file.text().then((text) => { - let table = loadTextDataWrapper(uniqueName, text, file.type); - if (table) { - dispatch(dfActions.loadTable(table)); - dispatch(fetchFieldSemanticType(table)); - } - }); - } else if (file.type === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' || - file.type === 'application/vnd.ms-excel' || - file.name.endsWith('.xlsx') || - file.name.endsWith('.xls')) { - // Handle Excel files - const reader = new FileReader(); - reader.onload = async (e) => { - const arrayBuffer = e.target?.result as ArrayBuffer; - if (arrayBuffer) { - try { - let tables = await loadBinaryDataWrapper(uniqueName, arrayBuffer); - for (let table of tables) { - dispatch(dfActions.loadTable(table)); - dispatch(fetchFieldSemanticType(table)); - } - if (tables.length == 0) { - dispatch(dfActions.addMessages({ - "timestamp": Date.now(), - "type": "error", - "component": "data loader", - "value": `Failed to parse Excel file ${file.name}. Please check the file format.` - })); - } - } catch (error) { - console.error('Error processing Excel file:', error); - dispatch(dfActions.addMessages({ - "timestamp": Date.now(), - "type": "error", - "component": "data loader", - "value": `Failed to parse Excel file ${file.name}. Please check the file format.` - })); - } - } - }; - reader.readAsArrayBuffer(file); - } else { - // Unsupported file type - dispatch(dfActions.addMessages({ - "timestamp": Date.now(), - "type": "error", - "component": "data loader", - "value": `Unsupported file format: ${file.name}. Please use CSV, TSV, JSON, or Excel files.` - })); - } - } - } - if (inputRef.current) { - inputRef.current.value = ''; - } - }; - - return ( - <> - - {buttonElement && ( - - Install Data Formulator locally to enable file upload.
    - Link: e.stopPropagation()} - > - https://github.com/microsoft/data-formulator - - - ) : ""} - placement="top" - > - - - -
    - )} - - ); -} - - -export interface TableCopyDialogProps { - buttonElement?: any; - disabled?: boolean; - onOpen?: () => void; - // Controlled mode props - open?: boolean; - onClose?: () => void; -} - -export interface TableURLDialogProps { - buttonElement: any; - disabled: boolean; -} - -export const TableURLDialog: React.FC = ({ buttonElement, disabled }) => { - - const [dialogOpen, setDialogOpen] = useState(false); - const [tableURL, setTableURL] = useState(""); - - const dispatch = useDispatch(); - - let handleSubmitContent = (): void => { - - let parts = tableURL.split('/'); - - // Get the last part of the URL, which should be the file name with extension - const tableName = parts[parts.length - 1]; - - fetch(tableURL) - .then(res => res.text()) - .then(content => { - let table : undefined | DictTable = undefined; - try { - let jsonContent = JSON.parse(content); - table = createTableFromFromObjectArray(tableName || 'dataset', jsonContent, true); - } catch (error) { - table = createTableFromText(tableName || 'dataset', content); - } - - if (table) { - dispatch(dfActions.loadTable(table)); - dispatch(fetchFieldSemanticType(table)); - } - }) - }; - - let hasValidSuffix = tableURL.endsWith('.csv') || tableURL.endsWith('.tsv') || tableURL.endsWith(".json"); - - let dialog = {setDialogOpen(false)}} open={dialogOpen} - sx={{ '& .MuiDialog-paper': { maxWidth: '80%', maxHeight: 800, minWidth: 800 } }} disableRestoreFocus - > - Upload data URL - { setDialogOpen(false); }} - aria-label="close" - > - - - - - { setTableURL(event.target.value.trim()); }} - id="dataset-url" label="data url" variant="outlined" /> - - - - - - ; - - return <> - - {dialog} - ; -} - - -export const TableCopyDialogV2: React.FC = ({ - buttonElement, - disabled, - onOpen, - open: controlledOpen, - onClose, -}) => { - - const [internalOpen, setInternalOpen] = useState(false); - - // Support both controlled and uncontrolled modes - const isControlled = controlledOpen !== undefined; - const dialogOpen = isControlled ? controlledOpen : internalOpen; - - const [tableContent, setTableContent] = useState(""); - const [tableContentType, setTableContentType] = useState<'text' | 'image'>('text'); - - const [cleaningInProgress, setCleaningInProgress] = useState(false); - - // Add new state for display optimization - const [displayContent, setDisplayContent] = useState(""); - const [isLargeContent, setIsLargeContent] = useState(false); - const [showFullContent, setShowFullContent] = useState(false); - const [isOverSizeLimit, setIsOverSizeLimit] = useState(false); - - // Constants for content size limits - const MAX_DISPLAY_LINES = 20; // Reduced from 30 - const LARGE_CONTENT_THRESHOLD = 50000; // ~50KB threshold - const MAX_CONTENT_SIZE = 2 * 1024 * 1024; // 2MB in bytes (same as file upload limit) - - const dispatch = useDispatch(); - const existingTables = useSelector((state: DataFormulatorState) => state.tables); - const existingNames = new Set(existingTables.map(t => t.id)); - - let handleSubmitContent = (tableStr: string): void => { - let table: undefined | DictTable = undefined; - - // Generate a short unique name based on content and time if no name provided - const defaultName = (() => { - const hashStr = tableStr.substring(0, 100) + Date.now(); - const hashCode = hashStr.split('').reduce((acc, char) => { - return ((acc << 5) - acc) + char.charCodeAt(0) | 0; - }, 0); - const shortHash = Math.abs(hashCode).toString(36).substring(0, 4); - return `data-${shortHash}`; - })(); - - const baseName = defaultName; - const uniqueName = getUniqueTableName(baseName, existingNames); - - try { - let content = JSON.parse(tableStr); - table = createTableFromFromObjectArray(uniqueName, content, true); - } catch (error) { - table = createTableFromText(uniqueName, tableStr); - } - if (table) { - dispatch(dfActions.loadTable(table)); - dispatch(fetchFieldSemanticType(table)); - } - }; - - // Optimized content change handler - const handleContentChange = useCallback((event: React.ChangeEvent) => { - const newContent = event.target.value; - setTableContent(newContent); - - // Check if content exceeds size limit - const contentSizeBytes = new Blob([newContent]).size; - const isOverLimit = contentSizeBytes > MAX_CONTENT_SIZE; - setIsOverSizeLimit(isOverLimit); - - // Check if content is large - const isLarge = newContent.length > LARGE_CONTENT_THRESHOLD; - setIsLargeContent(isLarge); - - if (isLarge && !showFullContent) { - // For large content, only show a preview in the TextField - const lines = newContent.split('\n'); - const previewLines = lines.slice(0, MAX_DISPLAY_LINES); - const preview = previewLines.join('\n') + (lines.length > MAX_DISPLAY_LINES ? '\n... (truncated for performance)' : ''); - setDisplayContent(preview); - } else { - setDisplayContent(newContent); - } - }, [showFullContent, dispatch, MAX_CONTENT_SIZE]); - - // Toggle between preview and full content - const toggleFullContent = useCallback(() => { - setShowFullContent(!showFullContent); - if (!showFullContent) { - setDisplayContent(tableContent); - } else { - const lines = tableContent.split('\n'); - const previewLines = lines.slice(0, MAX_DISPLAY_LINES); - const preview = previewLines.join('\n') + (lines.length > MAX_DISPLAY_LINES ? '\n... (truncated for performance)' : ''); - setDisplayContent(preview); - } - }, [showFullContent, tableContent]); - - - const handleCloseDialog = useCallback(() => { - if (isControlled) { - onClose?.(); - } else { - setInternalOpen(false); - } - // Reset state when closing - setTableContent(""); - setDisplayContent(""); - setIsLargeContent(false); - setIsOverSizeLimit(false); - setShowFullContent(false); - }, [isControlled, onClose]); - - let dialog = - Paste & Upload Data - - - - - - - {cleaningInProgress && tableContentType == "text" ? : ""} - - {/* Size limit warning */} - {isOverSizeLimit && ( - - - ⚠️ Content exceeds {(MAX_CONTENT_SIZE / (1024 * 1024)).toFixed(0)}MB size limit. - Current size: {(new Blob([tableContent]).size / (1024 * 1024)).toFixed(2)}MB. - Please use the DATABASE option for large datasets. - - - )} - {/* Content size indicator */} - {isLargeContent && !isOverSizeLimit && ( - - - Large content detected ({Math.round(tableContent.length / 1000)}KB). - {showFullContent ? 'Showing full content (may be slow)' : 'Showing preview for performance'} - - - - )} - - { - if (e.clipboardData.files.length > 0) { - let file = e.clipboardData.files[0]; - let read = new FileReader(); - - read.readAsDataURL(file); - read.onloadend = function(){ - let res = read.result; - console.log(res); - if (res) { - setTableContent(res as string); - setTableContentType("image"); - } - } - } - }} - autoComplete='off' - label="data content" - variant="outlined" - multiline - /> - - - - - - - - - - - - - ; - - return <> - {buttonElement && ( - - )} - {dialog} - ; -} - diff --git a/src/views/UnifiedDataUploadDialog.tsx b/src/views/UnifiedDataUploadDialog.tsx new file mode 100644 index 0000000..82be5f0 --- /dev/null +++ b/src/views/UnifiedDataUploadDialog.tsx @@ -0,0 +1,1221 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import * as React from 'react'; +import { useState, useCallback, useEffect, useRef } from 'react'; +import { + Box, + Button, + Chip, + Dialog, + DialogContent, + DialogTitle, + IconButton, + TextField, + Typography, + Tooltip, + LinearProgress, + Link, + Input, + alpha, + useTheme, + Card, +} from '@mui/material'; + +import CloseIcon from '@mui/icons-material/Close'; +import UploadFileIcon from '@mui/icons-material/UploadFile'; +import ContentPasteIcon from '@mui/icons-material/ContentPaste'; +import LinkIcon from '@mui/icons-material/Link'; +import StorageIcon from '@mui/icons-material/Storage'; +import ImageSearchIcon from '@mui/icons-material/ImageSearch'; +import ExploreIcon from '@mui/icons-material/Explore'; +import RestartAltIcon from '@mui/icons-material/RestartAlt'; +import DeleteIcon from '@mui/icons-material/Delete'; +import ArrowBackIcon from '@mui/icons-material/ArrowBack'; +import Paper from '@mui/material/Paper'; + +import { useDispatch, useSelector } from 'react-redux'; +import { DataFormulatorState, dfActions, fetchFieldSemanticType } from '../app/dfSlice'; +import { AppDispatch } from '../app/store'; +import { DictTable } from '../components/ComponentType'; +import { createTableFromFromObjectArray, createTableFromText, loadTextDataWrapper, loadBinaryDataWrapper } from '../data/utils'; +import { DataLoadingChat } from './DataLoadingChat'; +import { DatasetSelectionView, DatasetMetadata } from './TableSelectionView'; +import { getUrls } from '../app/utils'; +import { CustomReactTable } from './ReactTable'; +import { DBManagerPane } from './DBTableManager'; + +export type UploadTabType = 'menu' | 'upload' | 'paste' | 'url' | 'database' | 'extract' | 'explore'; + +interface TabPanelProps { + children?: React.ReactNode; + index: UploadTabType; + value: UploadTabType; +} + +function TabPanel(props: TabPanelProps) { + const { children, value, index, ...other } = props; + + return ( + + ); +} + +// Data source menu card component +interface DataSourceCardProps { + icon: React.ReactNode; + title: string; + description: string; + onClick: () => void; + disabled?: boolean; + disabledReason?: string; +} + +interface PreviewPanelProps { + title: string; + loading: boolean; + error: string | null; + table?: DictTable | null; + tables?: DictTable[] | null; + emptyLabel: string; + meta?: string; + onRemoveTable?: (index: number) => void; + activeIndex?: number; + onActiveIndexChange?: (index: number) => void; +} + +const PreviewPanel: React.FC = ({ + title, + loading, + error, + table, + tables, + emptyLabel, + meta, + onRemoveTable, + activeIndex: controlledActiveIndex, + onActiveIndexChange, +}) => { + const previewTables = tables ?? (table ? [table] : null); + const [internalActiveIndex, setInternalActiveIndex] = useState(0); + const activeIndex = controlledActiveIndex !== undefined ? controlledActiveIndex : internalActiveIndex; + const setActiveIndex = onActiveIndexChange || setInternalActiveIndex; + + useEffect(() => { + if (!previewTables || previewTables.length === 0) { + if (onActiveIndexChange) { + onActiveIndexChange(0); + } else { + setInternalActiveIndex(0); + } + return; + } + if (activeIndex > previewTables.length - 1) { + const newIndex = previewTables.length - 1; + if (onActiveIndexChange) { + onActiveIndexChange(newIndex); + } else { + setInternalActiveIndex(newIndex); + } + } + }, [previewTables, activeIndex, onActiveIndexChange]); + + const activeTable = previewTables && previewTables.length > 0 ? previewTables[activeIndex] : null; + return ( + + {loading && } + + {error && ( + + {error} + + )} + + {!loading && !error && (!previewTables || previewTables.length === 0) && ( + + {emptyLabel} + + )} + + {previewTables && previewTables.length > 0 && ( + + + Preview + {previewTables.map((t, idx) => { + const label = t.displayId || t.id; + const isSelected = idx === activeIndex; + return ( + + { + if (onActiveIndexChange) { + onActiveIndexChange(idx); + } else { + setInternalActiveIndex(idx); + } + }} + sx={{ + borderRadius: 1, + cursor: 'pointer', + maxWidth: 160, + backgroundColor: (theme) => + isSelected + ? alpha(theme.palette.primary.main, 0.12) + : 'transparent', + borderColor: (theme) => + isSelected + ? alpha(theme.palette.primary.main, 0.5) + : undefined, + color: (theme) => + isSelected + ? theme.palette.primary.main + : theme.palette.text.secondary, + '& .MuiChip-label': { + fontSize: '0.625rem', + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap', + maxWidth: 140, + }, + '&:hover': { + backgroundColor: (theme) => + isSelected + ? alpha(theme.palette.primary.main, 0.16) + : alpha(theme.palette.primary.main, 0.08), + }, + }} + /> + + ); + })} + {onRemoveTable && ( + onRemoveTable(activeIndex)} + sx={{ ml: 'auto', flexShrink: 0 }} + aria-label="Remove table" + > + + + )} + + + {activeTable && ( + + + ({ + id: name, + label: name, + minWidth: 60, + }))} + rowsPerPageNum={-1} + compact={true} + isIncompleteTable={activeTable.rows.length > 12} + maxHeight={200} + /> + + + + {activeTable.rows.length} rows × {activeTable.names.length} columns + + + )} + + )} + + ); +}; + +const DataSourceCard: React.FC = ({ + icon, + title, + description, + onClick, + disabled = false, + disabledReason +}) => { + const theme = useTheme(); + + const card = ( + + + {icon} + + + + {title} + + + {description} + + + + ); + + if (disabled && disabledReason) { + return ( + + {card} + + ); + } + + return card; +}; + +const getUniqueTableName = (baseName: string, existingNames: Set): string => { + let uniqueName = baseName; + let counter = 1; + while (existingNames.has(uniqueName)) { + uniqueName = `${baseName}_${counter}`; + counter++; + } + return uniqueName; +}; + +export interface UnifiedDataUploadDialogProps { + open: boolean; + onClose: () => void; + initialTab?: UploadTabType; +} + +export const UnifiedDataUploadDialog: React.FC = ({ + open, + onClose, + initialTab = 'menu', +}) => { + const theme = useTheme(); + const dispatch = useDispatch(); + const existingTables = useSelector((state: DataFormulatorState) => state.tables); + const serverConfig = useSelector((state: DataFormulatorState) => state.serverConfig); + const dataCleanBlocks = useSelector((state: DataFormulatorState) => state.dataCleanBlocks); + const existingNames = new Set(existingTables.map(t => t.id)); + + const [activeTab, setActiveTab] = useState(initialTab === 'menu' ? 'menu' : initialTab); + const fileInputRef = useRef(null); + + // Paste tab state + const [pasteContent, setPasteContent] = useState(""); + const [displayContent, setDisplayContent] = useState(""); + const [isLargeContent, setIsLargeContent] = useState(false); + const [showFullContent, setShowFullContent] = useState(false); + const [isOverSizeLimit, setIsOverSizeLimit] = useState(false); + + // URL input state (merged into file upload) + const [tableURL, setTableURL] = useState(""); + + // File preview state (shared with URL) + const [filePreviewTables, setFilePreviewTables] = useState(null); + const [filePreviewLoading, setFilePreviewLoading] = useState(false); + const [filePreviewError, setFilePreviewError] = useState(null); + const [filePreviewFiles, setFilePreviewFiles] = useState([]); + const [filePreviewActiveIndex, setFilePreviewActiveIndex] = useState(0); + + // Sample datasets state + const [datasetPreviews, setDatasetPreviews] = useState([]); + + // Constants + const MAX_DISPLAY_LINES = 20; + const LARGE_CONTENT_THRESHOLD = 50000; + const MAX_CONTENT_SIZE = 2 * 1024 * 1024; + + // Update active tab when initialTab changes + useEffect(() => { + if (open) { + setActiveTab(initialTab === 'menu' ? 'menu' : initialTab); + } + }, [initialTab, open]); + + + // Load sample datasets + useEffect(() => { + if (open && activeTab === 'explore') { + fetch(`${getUrls().EXAMPLE_DATASETS}`) + .then((response) => response.json()) + .then((result) => { + let datasets: DatasetMetadata[] = result.map((info: any) => { + let tables = info["tables"].map((table: any) => { + if (table["format"] == "json") { + return { + table_name: table["name"], + url: table["url"], + format: table["format"], + sample: table["sample"], + } + } + else if (table["format"] == "csv" || table["format"] == "tsv") { + const delimiter = table["format"] === "csv" ? "," : "\t"; + const rows = table["sample"] + .split("\n") + .map((row: string) => row.split(delimiter)); + + if (rows.length > 0) { + const headers = rows[0]; + const dataRows = rows.slice(1); + const sampleData = dataRows.map((row: string[]) => { + const obj: any = {}; + headers.forEach((header: string, index: number) => { + obj[header] = row[index] || ''; + }); + return obj; + }); + + return { + table_name: table["name"], + url: table["url"], + format: table["format"], + sample: sampleData, + }; + } + + return { + table_name: table["name"], + url: table["url"], + format: table["format"], + sample: [], + }; + } + }) + return { tables: tables, name: info["name"], description: info["description"], source: info["source"] } + }).filter((t: DatasetMetadata | undefined) => t != undefined); + setDatasetPreviews(datasets); + }); + } + }, [open, activeTab]); + + const handleClose = useCallback(() => { + // Reset state when closing + setPasteContent(""); + setDisplayContent(""); + setIsLargeContent(false); + setIsOverSizeLimit(false); + setShowFullContent(false); + setTableURL(""); + setFilePreviewTables(null); + setFilePreviewLoading(false); + setFilePreviewError(null); + setFilePreviewFiles([]); + onClose(); + }, [onClose]); + + // File upload handler + const handleFileUpload = (event: React.ChangeEvent): void => { + const files = event.target.files; + + if (files && files.length > 0) { + const selectedFiles = Array.from(files); + setFilePreviewFiles(selectedFiles); + setFilePreviewError(null); + setFilePreviewTables(null); + setFilePreviewLoading(true); + // Clear URL input when file is uploaded + setTableURL(""); + + const MAX_FILE_SIZE = 5 * 1024 * 1024; + const previewTables: DictTable[] = []; + const errors: string[] = []; + + const processFiles = async () => { + for (const file of selectedFiles) { + const uniqueName = getUniqueTableName(file.name, existingNames); + const isTextFile = file.type === 'text/csv' || + file.type === 'text/tab-separated-values' || + file.type === 'application/json' || + file.name.endsWith('.csv') || + file.name.endsWith('.tsv') || + file.name.endsWith('.json'); + const isExcelFile = file.type === 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' || + file.type === 'application/vnd.ms-excel' || + file.name.endsWith('.xlsx') || + file.name.endsWith('.xls'); + + if (file.size > MAX_FILE_SIZE && isTextFile) { + errors.push(`File ${file.name} is too large (${(file.size / (1024 * 1024)).toFixed(2)}MB). Use Database for large files.`); + continue; + } + + if (isTextFile) { + try { + const text = await file.text(); + const table = loadTextDataWrapper(uniqueName, text, file.type); + if (table) { + previewTables.push(table); + } else { + errors.push(`Failed to parse ${file.name}.`); + } + } catch { + errors.push(`Failed to read ${file.name}.`); + } + continue; + } + + if (isExcelFile) { + try { + const arrayBuffer = await file.arrayBuffer(); + const tables = await loadBinaryDataWrapper(uniqueName, arrayBuffer); + if (tables.length > 0) { + previewTables.push(...tables); + } else { + errors.push(`Failed to parse Excel file ${file.name}.`); + } + } catch { + errors.push(`Failed to parse Excel file ${file.name}.`); + } + continue; + } + + errors.push(`Unsupported file format: ${file.name}.`); + } + + setFilePreviewTables(previewTables.length > 0 ? previewTables : null); + setFilePreviewError(errors.length > 0 ? errors.join(' ') : null); + setFilePreviewLoading(false); + }; + + processFiles(); + } + if (fileInputRef.current) { + fileInputRef.current.value = ''; + } + }; + + // Reset activeIndex when tables change + useEffect(() => { + if (filePreviewTables && filePreviewTables.length > 0) { + if (filePreviewActiveIndex >= filePreviewTables.length) { + setFilePreviewActiveIndex(filePreviewTables.length - 1); + } + } else { + setFilePreviewActiveIndex(0); + } + }, [filePreviewTables, filePreviewActiveIndex]); + + const handleFileLoadSingleTable = (): void => { + if (!filePreviewTables || filePreviewTables.length === 0) { + return; + } + const table = filePreviewTables[filePreviewActiveIndex]; + if (table) { + dispatch(dfActions.loadTable(table)); + dispatch(fetchFieldSemanticType(table)); + handleClose(); + } + }; + + const handleFileLoadAllTables = (): void => { + if (!filePreviewTables || filePreviewTables.length === 0) { + return; + } + for (let table of filePreviewTables) { + dispatch(dfActions.loadTable(table)); + dispatch(fetchFieldSemanticType(table)); + } + handleClose(); + }; + + const handleRemoveFilePreviewTable = (index: number): void => { + setFilePreviewTables((prev) => { + if (!prev) return prev; + const next = prev.filter((_, i) => i !== index); + return next.length > 0 ? next : null; + }); + }; + + // Paste content handler + const handleContentChange = useCallback((event: React.ChangeEvent) => { + const newContent = event.target.value; + setPasteContent(newContent); + + const contentSizeBytes = new Blob([newContent]).size; + const isOverLimit = contentSizeBytes > MAX_CONTENT_SIZE; + setIsOverSizeLimit(isOverLimit); + + const isLarge = newContent.length > LARGE_CONTENT_THRESHOLD; + setIsLargeContent(isLarge); + + if (isLarge && !showFullContent) { + const lines = newContent.split('\n'); + const previewLines = lines.slice(0, MAX_DISPLAY_LINES); + const preview = previewLines.join('\n') + (lines.length > MAX_DISPLAY_LINES ? '\n... (truncated for performance)' : ''); + setDisplayContent(preview); + } else { + setDisplayContent(newContent); + } + }, [showFullContent, MAX_CONTENT_SIZE]); + + const toggleFullContent = useCallback(() => { + setShowFullContent(!showFullContent); + if (!showFullContent) { + setDisplayContent(pasteContent); + } else { + const lines = pasteContent.split('\n'); + const previewLines = lines.slice(0, MAX_DISPLAY_LINES); + const preview = previewLines.join('\n') + (lines.length > MAX_DISPLAY_LINES ? '\n... (truncated for performance)' : ''); + setDisplayContent(preview); + } + }, [showFullContent, pasteContent]); + + const handlePasteSubmit = (): void => { + let table: undefined | DictTable = undefined; + + const defaultName = (() => { + const hashStr = pasteContent.substring(0, 100) + Date.now(); + const hashCode = hashStr.split('').reduce((acc, char) => { + return ((acc << 5) - acc) + char.charCodeAt(0) | 0; + }, 0); + const shortHash = Math.abs(hashCode).toString(36).substring(0, 4); + return `data-${shortHash}`; + })(); + + const uniqueName = getUniqueTableName(defaultName, existingNames); + + try { + let content = JSON.parse(pasteContent); + table = createTableFromFromObjectArray(uniqueName, content, true); + } catch (error) { + table = createTableFromText(uniqueName, pasteContent); + } + if (table) { + dispatch(dfActions.loadTable(table)); + dispatch(fetchFieldSemanticType(table)); + handleClose(); + } + }; + + + const handleURLPreview = (): void => { + setFilePreviewLoading(true); + setFilePreviewError(null); + setFilePreviewTables(null); + // Clear file preview when URL is loaded + setFilePreviewFiles([]); + + let parts = tableURL.split('/'); + const baseName = parts[parts.length - 1] || 'dataset'; + const tableName = getUniqueTableName(baseName, existingNames); + + fetch(tableURL) + .then(res => res.text()) + .then(content => { + let table: undefined | DictTable = undefined; + try { + let jsonContent = JSON.parse(content); + if (!Array.isArray(jsonContent)) { + throw new Error('JSON content must be an array of objects.'); + } + table = createTableFromFromObjectArray(tableName, jsonContent, true); + } catch (error) { + table = createTableFromText(tableName, content); + } + + if (table) { + setFilePreviewTables([table]); + } else { + setFilePreviewError('Unable to parse data from the provided URL.'); + } + }) + .catch(() => { + setFilePreviewError('Failed to fetch data from the URL.'); + }) + .finally(() => { + setFilePreviewLoading(false); + }); + }; + + const hasValidUrlSuffix = tableURL.endsWith('.csv') || tableURL.endsWith('.tsv') || tableURL.endsWith(".json"); + const hasMultipleFileTables = (filePreviewTables?.length || 0) > 1; + const showFilePreview = filePreviewLoading || !!filePreviewError || (filePreviewTables && filePreviewTables.length > 0); + const hasPasteContent = pasteContent.trim() !== ''; + + // Data source configurations for the menu + const regularDataSources = [ + { + value: 'explore' as UploadTabType, + title: 'Sample Datasets', + description: 'Explore and load curated example datasets', + icon: , + disabled: false, + disabledReason: undefined + }, + { + value: 'upload' as UploadTabType, + title: 'Upload File', + description: 'Structured data (CSV, TSV, JSON, Excel) from files or URLs', + icon: , + disabled: false, + disabledReason: undefined + }, + { + value: 'paste' as UploadTabType, + title: 'Paste Data', + description: 'Paste tabular data directly from clipboard', + icon: , + disabled: false, + disabledReason: undefined + }, + { + value: 'extract' as UploadTabType, + title: 'Extract from Documents', + description: 'Extract tables from images, PDFs, or documents using AI', + icon: , + disabled: false, + disabledReason: undefined + }, + ]; + + const databaseDataSources = [ + { + value: 'database' as UploadTabType, + title: 'Database', + description: 'Connect to databases or data services', + icon: , + disabled: serverConfig.DISABLE_DATABASE, + disabledReason: 'Database connection is disabled in this environment' + }, + ]; + + // Combined config for finding tab titles + const dataSourceConfig = [...regularDataSources, ...databaseDataSources]; + + // Get current tab title for header + const getCurrentTabTitle = () => { + const tab = dataSourceConfig.find(t => t.value === activeTab); + return tab?.title || 'Add Data'; + }; + + return ( + + + {activeTab !== 'menu' && ( + setActiveTab('menu')} + sx={{ mr: 0.5 }} + > + + + )} + + {activeTab === 'menu' ? 'Load Data' : getCurrentTabTitle()} + + {activeTab === 'extract' && dataCleanBlocks.length > 0 && ( + + dispatch(dfActions.resetDataCleanBlocks())} + > + + + + )} + + + + + + + {/* Main Menu */} + + + + {/* Section hint */} + + Local data + + + {/* Regular Data Sources Group */} + + {regularDataSources.map((source) => ( + setActiveTab(source.value)} + disabled={source.disabled} + disabledReason={source.disabledReason} + /> + ))} + + + {/* Section hint */} + + Or connect to a database + + + {/* Database Data Sources Group */} + + {databaseDataSources.map((source) => ( + setActiveTab(source.value)} + disabled={source.disabled} + disabledReason={source.disabledReason} + /> + ))} + + + + + + {/* Upload File Tab */} + + + + + {serverConfig.DISABLE_FILE_UPLOAD ? ( + + + File upload is disabled in this environment. + + + Install Data Formulator locally to enable file upload.
    + + https://github.com/microsoft/data-formulator + +
    +
    + ) : ( + + fileInputRef.current?.click()} + > + + + Drag & drop file here + + + or Browse + + {!showFilePreview && ( + + Supported: CSV, TSV, JSON, Excel (xlsx, xls) + + )} + + + {/* URL Input Section */} + + setTableURL(e.target.value.trim())} + error={tableURL !== "" && !hasValidUrlSuffix} + size="small" + sx={{ + flex: 1, + '& .MuiInputBase-input': { + fontSize: '0.75rem', + }, + '& .MuiInputBase-input::placeholder': { + fontSize: '0.75rem', + }, + }} + /> + + + + )} +
    + + {showFilePreview && ( + + 0 ? `${filePreviewTables.length} table${filePreviewTables.length > 1 ? 's' : ''} previewed${hasMultipleFileTables ? ' • Multiple sheets detected' : ''}` : undefined} + onRemoveTable={handleRemoveFilePreviewTable} + activeIndex={filePreviewActiveIndex} + onActiveIndexChange={setFilePreviewActiveIndex} + /> + + )} + + {filePreviewTables && filePreviewTables.length > 0 && ( + + + {hasMultipleFileTables && ( + + )} + + )} +
    +
    + + {/* Paste Data Tab */} + + + {isOverSizeLimit && ( + + + ⚠️ Content exceeds {(MAX_CONTENT_SIZE / (1024 * 1024)).toFixed(0)}MB size limit. + Current size: {(new Blob([pasteContent]).size / (1024 * 1024)).toFixed(2)}MB. + Please use the DATABASE tab for large datasets. + + + )} + + {isLargeContent && !isOverSizeLimit && ( + + + Large content detected ({Math.round(pasteContent.length / 1000)}KB). + {showFullContent ? 'Showing full content (may be slow)' : 'Showing preview for performance'} + + + + )} + + + + + + + + + + + + {/* Database Tab */} + + {serverConfig.DISABLE_DATABASE ? ( + + + Database connection is disabled in this environment. + + + Install Data Formulator locally to use database features.
    + + https://github.com/microsoft/data-formulator + +
    +
    + ) : ( + + )} +
    + + {/* Extract Data Tab */} + + + + + {/* Explore Sample Datasets Tab */} + + + { + for (let table of dataset.tables) { + fetch(table.url) + .then(res => res.text()) + .then(textData => { + let tableName = table.url.split("/").pop()?.split(".")[0] || 'table-' + Date.now().toString().substring(0, 8); + let dictTable; + if (table.format == "csv") { + dictTable = createTableFromText(tableName, textData); + } else if (table.format == "json") { + dictTable = createTableFromFromObjectArray(tableName, JSON.parse(textData), true); + } + if (dictTable) { + dispatch(dfActions.loadTable(dictTable)); + dispatch(fetchFieldSemanticType(dictTable)); + } + }); + } + handleClose(); + }} + /> + + +
    +
    + ); +}; + +export default UnifiedDataUploadDialog; diff --git a/src/views/VisualizationView.tsx b/src/views/VisualizationView.tsx index 07f9819..fa55183 100644 --- a/src/views/VisualizationView.tsx +++ b/src/views/VisualizationView.tsx @@ -553,7 +553,7 @@ export const ChartEditorFC: FC<{}> = function ChartEditorFC({}) { setVisTableTotalRowCount(table.virtual?.rowCount || table.rows.length); setDataVersion(versionId); } - }, [dataRequirements]) + }, [dataRequirements, table.rows]) diff --git a/yarn.lock b/yarn.lock index f1d9bfb..069c709 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1134,7 +1134,7 @@ dependencies: dompurify "*" -"@types/estree@1.0.8", "@types/estree@^1.0.0", "@types/estree@^1.0.6", "@types/estree@^1.0.8": +"@types/estree@1.0.8", "@types/estree@^1.0.6", "@types/estree@^1.0.8": version "1.0.8" resolved "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz" integrity sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w== @@ -1367,10 +1367,10 @@ allotment@^1.20.4: lodash.isequal "^4.5.0" use-resize-observer "^9.0.0" -ansi-regex@^5.0.1: - version "5.0.1" - resolved "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz" - integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== +ansi-regex@^6.0.1: + version "6.2.2" + resolved "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz#60216eea464d864597ce2832000738a0589650c1" + integrity sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg== ansi-styles@^3.2.1: version "3.2.1" @@ -1379,13 +1379,18 @@ ansi-styles@^3.2.1: dependencies: color-convert "^1.9.0" -ansi-styles@^4.0.0, ansi-styles@^4.1.0: +ansi-styles@^4.1.0: version "4.3.0" resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz" integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== dependencies: color-convert "^2.0.1" +ansi-styles@^6.2.1: + version "6.2.3" + resolved "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz#c044d5dcc521a076413472597a1acb1f103c4041" + integrity sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg== + archiver-utils@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz" @@ -1733,14 +1738,14 @@ classnames@^2.3.0: resolved "https://registry.npmjs.org/classnames/-/classnames-2.5.1.tgz" integrity sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow== -cliui@^8.0.1: - version "8.0.1" - resolved "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz" - integrity sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ== +cliui@^9.0.1: + version "9.0.1" + resolved "https://registry.npmjs.org/cliui/-/cliui-9.0.1.tgz#6f7890f386f6f1f79953adc1f78dec46fcc2d291" + integrity sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w== dependencies: - string-width "^4.2.0" - strip-ansi "^6.0.1" - wrap-ansi "^7.0.0" + string-width "^7.2.0" + strip-ansi "^7.1.0" + wrap-ansi "^9.0.0" clsx@^2.0.0, clsx@^2.1.1: version "2.1.1" @@ -2320,10 +2325,10 @@ duplexer2@~0.1.4: dependencies: readable-stream "^2.0.2" -emoji-regex@^8.0.0: - version "8.0.0" - resolved "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz" - integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== +emoji-regex@^10.3.0: + version "10.6.0" + resolved "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz#bf3d6e8f7f8fd22a65d9703475bc0147357a6b0d" + integrity sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A== emoji-regex@^9.2.2: version "9.2.2" @@ -2824,6 +2829,11 @@ get-caller-file@^2.0.5: resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== +get-east-asian-width@^1.0.0: + version "1.4.0" + resolved "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.4.0.tgz#9bc4caa131702b4b61729cb7e42735bc550c9ee6" + integrity sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q== + get-intrinsic@^1.2.4, get-intrinsic@^1.2.5, get-intrinsic@^1.2.6, get-intrinsic@^1.2.7, get-intrinsic@^1.3.0: version "1.3.0" resolved "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz" @@ -3147,11 +3157,6 @@ is-finalizationregistry@^1.1.0: dependencies: call-bound "^1.0.3" -is-fullwidth-code-point@^3.0.0: - version "3.0.0" - resolved "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz" - integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== - is-generator-function@^1.0.10: version "1.1.2" resolved "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.1.2.tgz" @@ -4171,11 +4176,6 @@ regexp.prototype.flags@^1.5.3, regexp.prototype.flags@^1.5.4: gopd "^1.2.0" set-function-name "^2.0.2" -require-directory@^2.1.1: - version "2.1.1" - resolved "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz" - integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q== - reselect@^4.1.8: version "4.1.8" resolved "https://registry.npmjs.org/reselect/-/reselect-4.1.8.tgz" @@ -4458,14 +4458,14 @@ stop-iteration-iterator@^1.1.0: es-errors "^1.3.0" internal-slot "^1.1.0" -string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: - version "4.2.3" - resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== +string-width@^7.0.0, string-width@^7.2.0: + version "7.2.0" + resolved "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz#b5bb8e2165ce275d4d43476dd2700ad9091db6dc" + integrity sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ== dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" + emoji-regex "^10.3.0" + get-east-asian-width "^1.0.0" + strip-ansi "^7.1.0" string.prototype.includes@^2.0.1: version "2.0.1" @@ -4549,12 +4549,12 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== +strip-ansi@^7.1.0: + version "7.1.2" + resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.2.tgz#132875abde678c7ea8d691533f2e7e22bb744dba" + integrity sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA== dependencies: - ansi-regex "^5.0.1" + ansi-regex "^6.0.1" strip-json-comments@^3.1.1: version "3.1.1" @@ -4801,10 +4801,10 @@ uuid@^8.3.0: resolved "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz" integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== -validator@^13.15.22: - version "13.15.22" - resolved "https://registry.yarnpkg.com/validator/-/validator-13.15.22.tgz#5f847cf4a799107e5716fc87e5cf2a337a71eb14" - integrity sha512-uT/YQjiyLJP7HSrv/dPZqK9L28xf8hsNca01HSz1dfmI0DgMfjopp1rO/z13NeGF1tVystF0Ejx3y4rUKPw+bQ== +validator@^13.15.20: + version "13.15.23" + resolved "https://registry.npmjs.org/validator/-/validator-13.15.23.tgz#59a874f84e4594588e3409ab1edbe64e96d0c62d" + integrity sha512-4yoz1kEWqUjzi5zsPbAS/903QXSYp0UOtHsPpp7p9rHAw/W+dkInskAE386Fat3oKRROwO98d9ZB0G4cObgUyw== vega-canvas@^2.0.0: version "2.0.0" @@ -4871,11 +4871,6 @@ vega-event-selector@^4.0.0, vega-event-selector@~4.0.0: resolved "https://registry.yarnpkg.com/vega-event-selector/-/vega-event-selector-4.0.0.tgz#425e9f2671e858a1a45b4b6a7fc452ca0b22abbf" integrity sha512-CcWF4m4KL/al1Oa5qSzZ5R776q8lRxCj3IafCHs5xipoEHrkgu1BWa7F/IH5HrDNXeIDnqOpSV1pFsAWRak4gQ== -vega-event-selector@~3.0.1: - version "3.0.1" - resolved "https://registry.npmjs.org/vega-event-selector/-/vega-event-selector-3.0.1.tgz" - integrity sha512-K5zd7s5tjr1LiOOkjGpcVls8GsH/f2CWCrWcpKy74gTCp+llCdwz0Enqo013ZlGaRNjfgD/o1caJRt3GSaec4A== - vega-expression@^6.1.0, vega-expression@~6.1.0: version "6.1.0" resolved "https://registry.yarnpkg.com/vega-expression/-/vega-expression-6.1.0.tgz#6ce358a39b9b953806bff200f6f84f44163c9e38" @@ -4884,14 +4879,6 @@ vega-expression@^6.1.0, vega-expression@~6.1.0: "@types/estree" "^1.0.8" vega-util "^2.1.0" -vega-expression@~5.1.1: - version "5.1.2" - resolved "https://registry.npmjs.org/vega-expression/-/vega-expression-5.1.2.tgz" - integrity sha512-fFeDTh4UtOxlZWL54jf1ZqJHinyerWq/ROiqrQxqLkNJRJ86RmxYTgXwt65UoZ/l4VUv9eAd2qoJeDEf610Umw== - dependencies: - "@types/estree" "^1.0.0" - vega-util "^1.17.3" - vega-force@~5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/vega-force/-/vega-force-5.1.0.tgz#aa7cf8edbe2ae3bada070f343565dfb841e501a9" @@ -4969,17 +4956,17 @@ vega-label@~2.1.0: vega-scenegraph "^5.1.0" vega-util "^2.1.0" -vega-lite@^5.5.0: - version "5.23.0" - resolved "https://registry.npmjs.org/vega-lite/-/vega-lite-5.23.0.tgz" - integrity sha512-l4J6+AWE3DIjvovEoHl2LdtCUkfm4zs8Xxx7INwZEAv+XVb6kR6vIN1gt3t2gN2gs/y4DYTs/RPoTeYAuEg6mA== +vega-lite@6.4.1: + version "6.4.1" + resolved "https://registry.npmjs.org/vega-lite/-/vega-lite-6.4.1.tgz#549634ecaefd46d00f17e7922577d0c97a4663c5" + integrity sha512-KO3ybHNouRK4A0al/+2fN9UqgTEfxrd/ntGLY933Hg5UOYotDVQdshR3zn7OfXwQ7uj0W96Vfa5R+QxO8am3IQ== dependencies: json-stringify-pretty-compact "~4.0.0" tslib "~2.8.1" - vega-event-selector "~3.0.1" - vega-expression "~5.1.1" - vega-util "~1.17.2" - yargs "~17.7.2" + vega-event-selector "~4.0.0" + vega-expression "~6.1.0" + vega-util "~2.1.0" + yargs "~18.0.0" vega-loader@^5.1.0, vega-loader@~5.1.0: version "5.1.0" @@ -5130,7 +5117,7 @@ vega-typings@~2.1.0: vega-expression "^6.1.0" vega-util "^2.1.0" -vega-util@^1.13.1, vega-util@^1.17.2, vega-util@^1.17.3, vega-util@^1.17.4, vega-util@~1.17.2: +vega-util@^1.13.1, vega-util@^1.17.2, vega-util@^1.17.4: version "1.17.4" resolved "https://registry.npmjs.org/vega-util/-/vega-util-1.17.4.tgz" integrity sha512-+y3ZW7dEqM8Ck+KRsd+jkMfxfE7MrQxUyIpNjkfhIpGEreym+aTn7XUw1DKXqclr8mqTQvbilPo16B3lnBr0wA== @@ -5332,14 +5319,14 @@ word-wrap@^1.2.5: resolved "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz" integrity sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA== -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== +wrap-ansi@^9.0.0: + version "9.0.2" + resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz#956832dea9494306e6d209eb871643bb873d7c98" + integrity sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww== dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" + ansi-styles "^6.2.1" + string-width "^7.0.0" + strip-ansi "^7.1.0" wrappy@1: version "1.0.2" @@ -5361,23 +5348,22 @@ yaml@^1.10.0: resolved "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz" integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg== -yargs-parser@^21.1.1: - version "21.1.1" - resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz" - integrity sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw== +yargs-parser@^22.0.0: + version "22.0.0" + resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz#87b82094051b0567717346ecd00fd14804b357c8" + integrity sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw== -yargs@~17.7.2: - version "17.7.2" - resolved "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz" - integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== +yargs@~18.0.0: + version "18.0.0" + resolved "https://registry.npmjs.org/yargs/-/yargs-18.0.0.tgz#6c84259806273a746b09f579087b68a3c2d25bd1" + integrity sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg== dependencies: - cliui "^8.0.1" + cliui "^9.0.1" escalade "^3.1.1" get-caller-file "^2.0.5" - require-directory "^2.1.1" - string-width "^4.2.3" + string-width "^7.2.0" y18n "^5.0.5" - yargs-parser "^21.1.1" + yargs-parser "^22.0.0" yocto-queue@^0.1.0: version "0.1.0"