+ + {/* Header with title */} + + {toolName} + + Explore data with visualizations, powered by AI agents. + - - Explore data with visualizations, powered by AI agents. - {actionButtons} - {/* Interactive Features Carousel */} - - - {/* Left Arrow */} - + {features.map((feature, index) => ( + - - - - {/* Feature Content */} - {/* Text Content */} - {features[currentFeature].title} + {feature.title} - {features[currentFeature].description} + {feature.description} - - {/* Feature Indicators */} - - {features.map((_, index) => ( - setCurrentFeature(index)} - sx={{ - width: 32, - height: 4, - borderRadius: 2, - bgcolor: index === currentFeature - ? theme.palette.primary.main - : alpha(theme.palette.text.secondary, 0.2), - cursor: 'pointer', - '&:hover': { - bgcolor: index === currentFeature - ? theme.palette.primary.main - : alpha(theme.palette.text.secondary, 0.4), - } - }} - /> - ))} - {/* Media Content */} @@ -292,30 +177,18 @@ export const About: FC<{}> = function About({ }) { flex: 1, borderRadius: 2, overflow: 'hidden', - boxShadow: '0 4px 12px rgba(0,0,0,0.1)', - minWidth: 300, - maxWidth: 500, + border: '1px solid rgba(0,0,0,0.1)', }}> - {features[currentFeature].mediaType === 'video' ? ( + {feature.mediaType === 'video' ? ( { - const video = e.currentTarget as HTMLVideoElement; - if (video.duration && !isNaN(video.duration)) { - videoDurationsRef.current.set( - features[currentFeature].media, - video.duration - ); - } - }} + aria-label={`Video demonstration: ${feature.title}`} sx={{ width: '100%', height: 'auto', @@ -325,8 +198,8 @@ export const About: FC<{}> = function About({ }) { ) : ( = function About({ }) { )} - - {/* Right Arrow */} - - - - - - - {/* Screenshots Carousel Section */} - - - {/* Screenshot Container */} - setCurrentScreenshot((currentScreenshot + 1) % screenshots.length)} - sx={{ - height: 680, - width: 'auto', - borderRadius: 8, - cursor: 'pointer', - overflow: 'hidden', - border: '1px solid rgba(0,0,0,0.1)', - boxShadow: '0 4px 12px rgba(0,0,0,0.1)', - position: 'relative', - display: 'flex', - justifyContent: 'center', - textDecoration: 'none', - animation: 'fadeSlideIn 0.1s ease-out', - '&:hover': { - boxShadow: '0 8px 24px rgba(0,0,0,0.2)', - '& .description-overlay': { - opacity: 1, - } - } - }} - > - - - - {screenshots[currentScreenshot].description} - - - - - {/* Screenshot Indicators */} - - {screenshots.map((_, index) => ( - setCurrentScreenshot(index)} - sx={{ - width: 32, - height: 4, - borderRadius: 2, - bgcolor: index === currentScreenshot - ? theme.palette.primary.main - : alpha(theme.palette.text.secondary, 0.2), - cursor: 'pointer', - transition: 'all 0.3s ease', - '&:hover': { - bgcolor: index === currentScreenshot - ? theme.palette.primary.main - : alpha(theme.palette.text.secondary, 0.4), - } - }} - /> - ))} - - + ))} - - - How does Data Formulator handle your data? - - -
  • Data Storage: Uploaded data (csv, xlsx, json, clipboard, messy data etc.) is stored in browser's local storage only
    • -
  • Data Processing: Local installation runs Python on your machine; online demo sends the data to server for data transformations (but not stored)
    • -
  • Database: Only available for locally installed Data Formulator (a DuckDB database file is created in temp directory to store data); not available in online demo
    • -
  • LLM Endpoints: Small data samples are sent to LLM endpoints along with the prompt. Use your trusted model provider if working with private data.
    • -     - - Research Prototype from Microsoft Research - -     + + Data handling: Data stored in browser only • Local install runs Python locally; online demo processes server-side (not stored) • LLM receives small samples with prompts + + + Research Prototype from Microsoft Research +     {/* Footer */} - + - + ) } diff --git a/src/views/ChartRecBox.tsx b/src/views/ChartRecBox.tsx index 943f833d..850631bd 100644 --- a/src/views/ChartRecBox.tsx +++ b/src/views/ChartRecBox.tsx @@ -69,111 +69,6 @@ export interface ChartRecBoxProps { sx?: SxProps; } -// Table selector component for ChartRecBox -const NLTableSelector: FC<{ - selectedTableIds: string[], - tables: DictTable[], - updateSelectedTableIds: (tableIds: string[]) => void, - requiredTableIds?: string[] -}> = ({ selectedTableIds, tables, updateSelectedTableIds, requiredTableIds = [] }) => { - const [anchorEl, setAnchorEl] = useState(null); - const open = Boolean(anchorEl); - - const handleClick = (event: React.MouseEvent) => { - setAnchorEl(event.currentTarget); - }; - - const handleClose = () => { - setAnchorEl(null); - }; - - const handleTableSelect = (table: DictTable) => { - if (!selectedTableIds.includes(table.id)) { - updateSelectedTableIds([...selectedTableIds, table.id]); - } - handleClose(); - }; - - return ( - - {selectedTableIds.map((tableId) => { - const isRequired = requiredTableIds.includes(tableId); - return ( - t.id == tableId)?.displayId} - size="small" - sx={{ - height: 16, - fontSize: '10px', - borderRadius: '2px', - bgcolor: isRequired ? 'rgba(25, 118, 210, 0.2)' : 'rgba(25, 118, 210, 0.1)', - color: 'rgba(0, 0, 0, 0.7)', - '& .MuiChip-label': { - pl: '4px', - pr: '6px' - } - }} - deleteIcon={isRequired ? undefined : } - onDelete={isRequired ? undefined : () => updateSelectedTableIds(selectedTableIds.filter(id => id !== tableId))} - /> - ); - })} - - - - - - - {tables - .filter(t => t.derive === undefined || t.anchored) - .map((table) => { - const isSelected = selectedTableIds.includes(table.id); - const isRequired = requiredTableIds.includes(table.id); - return ( - handleTableSelect(table)} - sx={{ - fontSize: '12px', - display: 'flex', - justifyContent: 'space-between', - alignItems: 'center' - }} - > - {table.displayId} - {isRequired && (required)} - - ); - }) - } - - - ); -}; - - - export const IdeaChip: FC<{ mini?: boolean, idea: {text?: string, questions?: string[], goal: string, difficulty: 'easy' | 'medium' | 'hard', type?: 'branch' | 'deep_dive'} @@ -357,17 +252,11 @@ export const ChartRecBox: FC = function ({ tableId, placeHolde const currentTable = tables.find(t => t.id === tableId); const availableTables = tables.filter(t => t.derive === undefined || t.anchored); - const [additionalTableIds, setAdditionalTableIds] = useState([]); - + // Combine the main tableId with additional selected tables - const selectedTableIds = currentTable?.derive ? [...currentTable.derive.source, ...additionalTableIds] : [tableId, ...additionalTableIds]; - - const handleTableSelectionChange = (newTableIds: string[]) => { - // Filter out the main tableId since it's always included - const additionalIds = newTableIds.filter(id => id !== tableId); - setAdditionalTableIds(additionalIds); - }; - + let selectedTableIds = currentTable?.derive ? [...currentTable.derive.source] : [tableId]; + selectedTableIds = [...selectedTableIds, ...availableTables.map(t => t.id).filter(id => !selectedTableIds.includes(id))]; + // Function to get a question from the list with cycling const getQuestion = (): string => { return mode === "agent" ? "let's explore something interesting about the data" : "show something interesting about the data"; @@ -1238,17 +1127,6 @@ export const ChartRecBox: FC = function ({ tableId, placeHolde }} /> )} - {showTableSelector && ( - - - - )} - = function ({ tableId, placeHolde "& .MuiInput-underline:not(.Mui-disabled):before": { borderBottom: 'none', }, - "& .MuiInput-underline:(.Mui-disabled):before": { + "& .MuiInput-underline.Mui-disabled:before": { + borderBottom: 'none', + }, + "& .MuiInput-underline.Mui-disabled:after": { borderBottom: 'none', }, "& .MuiInput-underline:after": { diff --git a/src/views/DBTableManager.tsx b/src/views/DBTableManager.tsx index da5bc8c4..b0da233e 100644 --- a/src/views/DBTableManager.tsx +++ b/src/views/DBTableManager.tsx @@ -1,5 +1,5 @@ // TableManager.tsx -import React, { useState, useEffect, FC } from 'react'; +import React, { useState, useEffect, useCallback, FC, useRef } from 'react'; import { Card, CardContent, @@ -9,13 +9,7 @@ import { Box, IconButton, Paper, - Tabs, - Tab, TextField, - Dialog, - DialogActions, - DialogContent, - DialogTitle, Divider, SxProps, Table, @@ -26,31 +20,41 @@ import { TableRow, CircularProgress, ButtonGroup, + ToggleButton, + ToggleButtonGroup, Tooltip, MenuItem, + Menu, Chip, Collapse, styled, - ToggleButtonGroup, - ToggleButton, useTheme, Link, - Checkbox + Popover, + Switch, + Slider, + FormControlLabel } from '@mui/material'; import DeleteIcon from '@mui/icons-material/Delete'; import CloseIcon from '@mui/icons-material/Close'; -import AnalyticsIcon from '@mui/icons-material/Analytics'; import ExpandLessIcon from '@mui/icons-material/ExpandLess'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import TableRowsIcon from '@mui/icons-material/TableRows'; import RefreshIcon from '@mui/icons-material/Refresh'; -import ArrowForwardIcon from '@mui/icons-material/ArrowForward'; import SearchIcon from '@mui/icons-material/Search'; +import StreamIcon from '@mui/icons-material/Stream'; +import Autocomplete from '@mui/material/Autocomplete'; -import { getUrls } from '../app/utils'; +// Type for table import configuration +type TableImportConfig = + | { mode: 'none' } + | { mode: 'full' } + | { mode: 'subset'; rowLimit: number; sortColumns: string[]; sortOrder: 'asc' | 'desc' }; + +import { getUrls, fetchWithSession } from '../app/utils'; import { CustomReactTable } from './ReactTable'; -import { DictTable } from '../components/ComponentType'; +import { DataSourceConfig, DictTable } from '../components/ComponentType'; import { Type } from '../data/types'; import { useDispatch, useSelector } from 'react-redux'; import { dfActions, dfSelectors, getSessionId } from '../app/dfSlice'; @@ -58,27 +62,68 @@ import { alpha } from '@mui/material'; import { DataFormulatorState } from '../app/dfSlice'; import { fetchFieldSemanticType } from '../app/dfSlice'; import { AppDispatch } from '../app/store'; -import Editor from 'react-simple-code-editor'; import Markdown from 'markdown-to-jsx'; -import Prism from 'prismjs' -import 'prismjs/components/prism-javascript' // Language -import 'prismjs/themes/prism.css'; //Example style, you can use another -import PrecisionManufacturingIcon from '@mui/icons-material/PrecisionManufacturing'; import CheckIcon from '@mui/icons-material/Check'; import MuiMarkdown from 'mui-markdown'; import CleaningServicesIcon from '@mui/icons-material/CleaningServices'; +import MoreVertIcon from '@mui/icons-material/MoreVert'; +import UploadFileIcon from '@mui/icons-material/UploadFile'; +import DownloadIcon from '@mui/icons-material/Download'; +import RestartAltIcon from '@mui/icons-material/RestartAlt'; +import StorageIcon from '@mui/icons-material/Storage'; +import CloudUploadIcon from '@mui/icons-material/CloudUpload'; +import SettingsIcon from '@mui/icons-material/Settings'; + +// Industry-standard database icons +const TableIcon: React.FC<{ sx?: SxProps }> = ({ sx }) => ( + + {/* Single rectangle with grid lines - standard table icon */} + + + + + + +); -export const handleDBDownload = async (sessionId: string) => { +const ViewIcon: React.FC<{ sx?: SxProps }> = ({ sx }) => ( + + {/* Two overlapping rectangles - standard view icon */} + + + +); + +export const handleDBDownload = async (sessionId: string, dispatch?: any) => { try { - const response = await fetch(getUrls().DOWNLOAD_DB_FILE, { - method: 'GET', - }); + // Use fetchWithSession which automatically handles session ID if missing + const response = await fetchWithSession( + getUrls().DOWNLOAD_DB_FILE, + { method: 'GET' }, + dispatch + ); // Check if the response is ok if (!response.ok) { const errorData = await response.json(); - throw new Error(errorData.error || 'Failed to download database file'); + throw new Error(errorData.error || errorData.message || 'Failed to download database file'); } // Get the blob directly from response @@ -88,7 +133,7 @@ export const handleDBDownload = async (sessionId: string) => { // Create a temporary link element const link = document.createElement('a'); link.href = url; - link.download = `df_${sessionId?.slice(0, 4)}.db`; + link.download = `df_${sessionId?.slice(0, 4) || 'db'}.db`; document.body.appendChild(link); // Trigger download @@ -111,6 +156,16 @@ interface DBTable { row_count: number; sample_rows: any[]; view_source: string | null; + // Source metadata for refreshable tables (from data loaders) + // Backend stores connection info; frontend manages refresh timing + source_metadata?: { + table_name: string; + data_loader_type: string; + data_loader_params: Record; + source_table_name?: string; + source_query?: string; + last_refreshed?: string; + } | null; } interface ColumnStatistics { @@ -126,116 +181,9 @@ interface ColumnStatistics { }; } -interface TableStatisticsViewProps { - tableName: string; - columnStats: ColumnStatistics[]; -} - -export class TableStatisticsView extends React.Component { - render() { - const { tableName, columnStats } = this.props; - - // Common styles for header cells - const headerCellStyle = { - backgroundColor: '#fff', - fontSize: 10, - color: "#333", - borderBottomColor: (theme: any) => theme.palette.primary.main, - borderBottomWidth: '1px', - borderBottomStyle: 'solid', - padding: '6px' - }; - - // Common styles for body cells - const bodyCellStyle = { - fontSize: 10, - padding: '6px' - }; - - return ( - - - - - - Column - Type - Count - Unique - Null - Min - Max - Avg - - - - {columnStats.map((stat, idx) => ( - - - {stat.column} - - - {stat.type} - - - {stat.statistics.count} - - - {stat.statistics.unique_count} - - - {stat.statistics.null_count} - - - {stat.statistics.min !== undefined ? stat.statistics.min : '-'} - - - {stat.statistics.max !== undefined ? stat.statistics.max : '-'} - - - {stat.statistics.avg !== undefined ? - Number(stat.statistics.avg).toFixed(2) : '-'} - - - ))} - -
    -     -     - ); - } -} -export const DBTableSelectionDialog: React.FC<{ - buttonElement?: any, - sx?: SxProps, - onOpen?: () => void, - // Controlled mode props - open?: boolean, - onClose?: () => void -}> = function DBTableSelectionDialog({ - buttonElement, - sx, - onOpen, - open: controlledOpen, - onClose, -}) { +export const DBManagerPane: React.FC<{ +}> = function DBManagerPane({ }) { const theme = useTheme(); @@ -243,17 +191,9 @@ export const DBTableSelectionDialog: React.FC<{ const sessionId = useSelector((state: DataFormulatorState) => state.sessionId); const tables = useSelector((state: DataFormulatorState) => state.tables); const serverConfig = useSelector((state: DataFormulatorState) => state.serverConfig); + const dataLoaderConnectParams = useSelector((state: DataFormulatorState) => state.dataLoaderConnectParams); + - const [internalOpen, setInternalOpen] = useState(false); - - // Support both controlled and uncontrolled modes - const isControlled = controlledOpen !== undefined; - const tableDialogOpen = isControlled ? controlledOpen : internalOpen; - const setTableDialogOpen = isControlled - ? (open: boolean) => { if (!open && onClose) onClose(); } - : setInternalOpen; - const [tableAnalysisMap, setTableAnalysisMap] = useState>({}); - // maps data loader type to list of param defs const [dataLoaderMetadata, setDataLoaderMetadata] = useState([]); const [selectedTabKey, setSelectedTabKey] = useState(""); + const [selectedDataLoader, setSelectedDataLoader] = useState(""); + const [connectorMenuAnchorEl, setConnectorMenuAnchorEl] = useState(null); const [isUploading, setIsUploading] = useState(false); + const [resetAnchorEl, setResetAnchorEl] = useState(null); + const [tableMenuAnchorEl, setTableMenuAnchorEl] = useState(null); + const [showViews, setShowViews] = useState(false); + const dbFileInputRef = useRef(null); + const menuButtonRef = useRef(null); + + // Watch/auto-refresh settings for the currently selected table + const [watchEnabled, setWatchEnabled] = useState(false); + const [watchInterval, setWatchInterval] = useState(600); + + // Reset watch settings when selected table changes + useEffect(() => { + setWatchEnabled(false); + setWatchInterval(600); + }, [selectedTabKey]); + + // Helper to format interval for display + const formatInterval = (seconds: number) => { + if (seconds < 60) return `${seconds}s`; + if (seconds < 3600) return `${Math.floor(seconds / 60)}m`; + return `${Math.floor(seconds / 3600)}h`; + }; let setSystemMessage = (content: string, severity: "error" | "warning" | "info" | "success") => { dispatch(dfActions.addMessages({ @@ -278,25 +242,29 @@ export const DBTableSelectionDialog: React.FC<{ }, []); useEffect(() => { - if (!selectedTabKey.startsWith("dataLoader:") && dbTables.length == 0) { - setSelectedTabKey(""); - } else if (!selectedTabKey.startsWith("dataLoader:") && dbTables.find(t => t.name === selectedTabKey) == undefined) { - setSelectedTabKey(dbTables[0].name); + if (selectedDataLoader === "") { + if (dbTables.length == 0) { + setSelectedTabKey(""); + } else if (dbTables.find(t => t.name === selectedTabKey) == undefined) { + setSelectedTabKey(dbTables[0]?.name || ""); + } } - }, [dbTables]); + }, [dbTables, selectedDataLoader]); // Fetch list of tables - const fetchTables = async () => { - if (serverConfig.DISABLE_DATABASE) return; + const fetchTables = async (): Promise => { + if (serverConfig.DISABLE_DATABASE) return undefined; try { - const response = await fetch(getUrls().LIST_TABLES); + const response = await fetchWithSession(getUrls().LIST_TABLES, { method: 'GET' }, dispatch); const data = await response.json(); if (data.status === 'success') { setDbTables(data.tables); + return data.tables; } } catch (error) { setSystemMessage('Failed to fetch tables, please check if the server is running', "error"); } + return undefined; }; const fetchDataLoaders = async () => { @@ -329,10 +297,10 @@ export const DBTableSelectionDialog: React.FC<{ try { setIsUploading(true); - const response = await fetch(getUrls().UPLOAD_DB_FILE, { + const response = await fetchWithSession(getUrls().UPLOAD_DB_FILE, { method: 'POST', body: formData - }); + }, dispatch); const data = await response.json(); if (data.status === 'success') { fetchTables(); // Refresh table list @@ -357,10 +325,10 @@ export const DBTableSelectionDialog: React.FC<{ try { setIsUploading(true); - const response = await fetch(getUrls().CREATE_TABLE, { + const response = await fetchWithSession(getUrls().CREATE_TABLE, { method: 'POST', body: formData - }); + }, dispatch); const data = await response.json(); if (data.status === 'success') { if (data.is_renamed) { @@ -383,9 +351,9 @@ export const DBTableSelectionDialog: React.FC<{ const handleDBReset = async () => { try { - const response = await fetch(getUrls().RESET_DB_FILE, { + const response = await fetchWithSession(getUrls().RESET_DB_FILE, { method: 'POST', - }); + }, dispatch); const data = await response.json(); if (data.status === 'success') { fetchTables(); @@ -405,13 +373,13 @@ export const DBTableSelectionDialog: React.FC<{ let deletedViews = []; for (let view of unreferencedViews) { try { - const response = await fetch(getUrls().DELETE_TABLE, { + const response = await fetchWithSession(getUrls().DELETE_TABLE, { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify({ table_name: view.name }) - }); + }, dispatch); const data = await response.json(); if (data.status === 'success') { deletedViews.push(view.name); @@ -438,13 +406,13 @@ export const DBTableSelectionDialog: React.FC<{ } try { - const response = await fetch(getUrls().DELETE_TABLE, { + const response = await fetchWithSession(getUrls().DELETE_TABLE, { method: 'POST', headers: { 'Content-Type': 'application/json', }, body: JSON.stringify({ table_name: tableName }) - }); + }, dispatch); const data = await response.json(); if (data.status === 'success') { fetchTables(); @@ -457,52 +425,7 @@ export const DBTableSelectionDialog: React.FC<{ } }; - // Handle data analysis - const handleAnalyzeData = async (tableName: string) => { - if (!tableName) return; - if (tableAnalysisMap[tableName]) return; - - console.log('Analyzing table:', tableName); - - try { - const response = await fetch(getUrls().GET_COLUMN_STATS, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ table_name: tableName }) - }); - const data = await response.json(); - if (data.status === 'success') { - console.log('Analysis results:', data); - // Update the analysis map with the new results - setTableAnalysisMap(prevMap => ({ - ...prevMap, - [tableName]: data.statistics - })); - } - } catch (error) { - console.error('Failed to analyze table data:', error); - setSystemMessage('Failed to analyze table data, please check if the server is running', "error"); - } - }; - - // Toggle analysis view - const toggleAnalysisView = (tableName: string) => { - if (tableAnalysisMap[tableName]) { - // If we already have analysis, remove it to show table data again - setTableAnalysisMap(prevMap => { - const newMap = { ...prevMap }; - delete newMap[tableName]; - return newMap; - }); - } else { - // If no analysis yet, fetch it - handleAnalyzeData(tableName); - } - }; - - const handleAddTableToDF = (dbTable: DBTable) => { + const handleAddTableToDF = (dbTable: DBTable, refreshSettings?: {autoRefresh: boolean, refreshIntervalSeconds: number}) => { const convertSqlTypeToAppType = (sqlType: string): Type => { // Convert SQL types to application types sqlType = sqlType.toUpperCase(); @@ -519,6 +442,19 @@ export const DBTableSelectionDialog: React.FC<{ } }; + // Build source config - backend stores connection details, frontend just manages refresh timing + const sourceMeta = dbTable.source_metadata; + const sourceConfig: DataSourceConfig = { + type: 'database', + databaseTable: dbTable.name, + // Frontend manages these refresh settings (from user selection) + autoRefresh: refreshSettings?.autoRefresh ?? false, + refreshIntervalSeconds: refreshSettings?.refreshIntervalSeconds ?? 60, + // Backend has connection info if source_metadata exists + canRefresh: sourceMeta != null, + lastRefreshed: Date.now() + }; + let table: DictTable = { id: dbTable.name, displayId: dbTable.name, @@ -538,48 +474,17 @@ export const DBTableSelectionDialog: React.FC<{ }, anchored: true, // by default, db tables are anchored createdBy: 'user', - attachedMetadata: '' + attachedMetadata: '', + source: sourceConfig } dispatch(dfActions.loadTable(table)); dispatch(fetchFieldSemanticType(table)); - setTableDialogOpen(false); } - const handleTabChange = (event: React.SyntheticEvent, newValue: string) => { - setSelectedTabKey(newValue); - }; useEffect(() => { - if (tableDialogOpen) { - fetchTables(); - } - }, [tableDialogOpen]); - - let importButton = (buttonElement: React.ReactNode) => { - return - - - - - } - - let exportButton = - - - - - + fetchTables(); + }, []); function uploadFileButton(element: React.ReactNode, buttonSx?: SxProps) { return ( @@ -605,81 +510,182 @@ export const DBTableSelectionDialog: React.FC<{ ); } - let hasDerivedViews = dbTables.filter(t => t.view_source !== null).length > 0; - - let dataLoaderPanel = - + let tableSelectionPanel = + {/* Recent Data Loaders */} + + + External Data Loaders + + + setSelectedDataLoader("file upload")} + sx={{ + fontSize: '0.7rem', + height: 20, + maxWidth: '100%', + borderColor: selectedDataLoader === "file upload" + ? theme.palette.secondary.main + : theme.palette.divider, + '& .MuiChip-label': { + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap' + }, + }} + /> + {Object.keys(dataLoaderMetadata ?? {}) + .map((dataLoaderType) => ( + setSelectedDataLoader(dataLoaderType)} + sx={{ + fontSize: '0.7rem', + height: 20, + maxWidth: '100%', + backgroundColor: selectedDataLoader === dataLoaderType + ? alpha(theme.palette.secondary.main, 0.2) + : 'transparent', + borderColor: selectedDataLoader === dataLoaderType + ? theme.palette.secondary.main + : theme.palette.divider, + '& .MuiChip-label': { + overflow: 'hidden', + textOverflow: 'ellipsis', + whiteSpace: 'nowrap' + }, + }} + /> + ))} + + + + - Data Connectors + Local DuckDB - - - {["file upload", ...Object.keys(dataLoaderMetadata ?? {})].map((dataLoaderType, i) => ( - - ))} - ; - - let tableSelectionPanel = - - - Data Tables - - - { - fetchTables(); - }}> - - - + { + fetchTables(); + setTableMenuAnchorEl(null); + }} + dense + > + + Refresh table list + + + { + dbFileInputRef.current?.click(); + setTableMenuAnchorEl(null); + }} + disabled={isUploading} + dense + > + + Import database file + + { + if (!isUploading && dbTables.length > 0) { + // fetchWithSession will automatically handle session ID if missing + handleDBDownload(sessionId ?? '', dispatch) + .catch(error => { + console.error('Failed to download database:', error); + setSystemMessage('Failed to download database file', "error"); + }); + } + setTableMenuAnchorEl(null); + }} + disabled={isUploading || dbTables.length === 0} + dense + > + + Export database file + + { + setTableMenuAnchorEl(null); + if (!isUploading && menuButtonRef.current) { + // Use setTimeout to ensure menu closes before popover opens + setTimeout(() => { + setResetAnchorEl(menuButtonRef.current); + }, 100); + } + }} + disabled={isUploading} + dense + sx={{ color: 'error.main' }} + > + + Reset database + + + + {dbTables.length == 0 && @@ -688,122 +694,165 @@ export const DBTableSelectionDialog: React.FC<{ } {/* Regular Tables */} - {dbTables.filter(t => t.view_source === null).map((t, i) => ( - - ))} + textTransform: "none", + width: '100%', + maxWidth: '100%', + justifyContent: 'flex-start', + textAlign: 'left', + borderRadius: 0, + py: 0.5, + px: 2, + color: (selectedTabKey === t.name && selectedDataLoader === "") ? 'primary.main' : 'text.secondary', + borderRight: (selectedTabKey === t.name && selectedDataLoader === "") ? 2 : 0, + minWidth: 0, + }} + startIcon={} + endIcon={isLoaded ? : null} + > + + {t.name} + + + ); + })} {/* Derived Views Section */} - {hasDerivedViews && ( + {dbTables.filter(t => t.view_source !== null).length > 0 && ( - - - Derived Views - - - t.view_source !== null).length === 0} - onClick={() => { - handleCleanDerivedViews(); - }}> - - - - - - {dbTables.filter(t => t.view_source !== null).map((t, i) => ( + - ))} + + t.view_source !== null && t.view_source !== undefined && !tables.some(t2 => t2.id === t.name)).length === 0} + sx={{ + padding: 0.5, + mr: 0.5, + '&:hover': { + backgroundColor: alpha(theme.palette.primary.main, 0.08), + } + }} + > + + + + + + + {dbTables.filter(t => t.view_source !== null).map((t, i) => { + return ( + + ); + })} + + )} - let tableView = - {/* Empty state */} - {selectedTabKey === '' && ( - - The database is empty, refresh the table list or import some data to get started. - - )} + let dataConnectorView = + {/* File upload */} - {selectedTabKey === 'dataLoader:file upload' && ( + {selectedDataLoader === 'file upload' && ( {uploadFileButton({isUploading ? 'uploading...' : 'upload a csv/tsv file to the local database'})} @@ -811,7 +860,7 @@ export const DBTableSelectionDialog: React.FC<{ {/* Data loader forms */} {dataLoaderMetadata && Object.entries(dataLoaderMetadata).map(([dataLoaderType, metadata]) => ( - selectedTabKey === 'dataLoader:' + dataLoaderType && ( + selectedDataLoader === dataLoaderType && ( { setIsUploading(false); fetchTables().then(() => { + // Switch back to tables view after import + setSelectedDataLoader(""); // Navigate to the first imported table after tables are fetched if (status === "success" && importedTables && importedTables.length > 0) { setSelectedTabKey(importedTables[0]); @@ -837,196 +888,256 @@ export const DBTableSelectionDialog: React.FC<{ ) ))} + ; + + let tableView = + {/* Empty state */} + {selectedTabKey === '' && ( + + The database is empty, refresh the table list or import some data to get started. + + )} {/* Table content */} {dbTables.map((t, i) => { if (selectedTabKey !== t.name) return null; const currentTable = t; - const showingAnalysis = tableAnalysisMap[currentTable.name] !== undefined; + return ( - - - - {showingAnalysis ? "column stats for " : "sample data from "} - - {currentTable.name} - - - ({currentTable.columns.length} columns × {currentTable.row_count} rows) - + + + {currentTable.view_source ? : } + + {currentTable.name} - - - handleDropTable(currentTable.name)} - title="Drop Table" - > - - - - {showingAnalysis ? ( - - ) : ( - { - return Object.fromEntries(Object.entries(row).map(([key, value]: [string, any]) => { - return [key, String(value)]; - })); - }).slice(0, 9)} - columnDefs={currentTable.columns.map(col => ({ + + handleDropTable(currentTable.name)} + title="Drop Table" + > + + + + + + + { + return Object.fromEntries( + currentTable.columns.map((col) => [col.name, String(row[col.name] ?? '')]) + ); + })} + columnDefs={currentTable.columns.map((col) => ({ id: col.name, label: col.name, - minWidth: 60 + minWidth: 80 }))} rowsPerPageNum={-1} compact={false} + maxCellWidth={80} isIncompleteTable={currentTable.row_count > 10} + maxHeight={340} /> + + {currentTable.row_count > 10 && ( + + + Showing first 9 rows of {currentTable.row_count} total rows + + )} - - + + {tables.some(t => t.id === currentTable.name) ? ( + + + + Loaded + + + ) : ( + + {/* Watch settings - only show for tables that can be refreshed */} + {currentTable.source_metadata && ( + + + setWatchEnabled(e.target.checked)} + size="small" + /> + } + label={ + + Watch Mode + + } + /> + {watchEnabled ? ( + + + check for updates every + + {[ + { seconds: 10, label: '10s' }, + { seconds: 30, label: '30s' }, + { seconds: 60, label: '1m' }, + { seconds: 300, label: '5m' }, + { seconds: 600, label: '10m' }, + { seconds: 1800, label: '30m' }, + { seconds: 3600, label: '1h' }, + { seconds: 86400, label: '24h' }, + ].map((opt) => ( + setWatchInterval(opt.seconds)} + sx={{ + cursor: 'pointer', + fontSize: '0.7rem', + height: 24, + }} + /> + ))} + + ) : + automatically check and refresh data from the database at regular intervals + } + + + )} + + + )} ); })} ; let mainContent = - - {/* Button navigation - similar to TableSelectionView */} - - - {/* External Data Loaders Section */} - {dataLoaderPanel} - {/* Available Tables Section */} - {tableSelectionPanel} - - - {importButton(Import)} - , - {exportButton} - or - - the backend database - + + + Reset backend database and delete all tables? This cannot be undone. + + + + + + + + + {/* Content area - show connector view if a connector is selected, otherwise show table view */} + + {selectedDataLoader !== "" ? dataConnectorView : tableView} + - {/* Content area - using conditional rendering instead of TabPanel */} - {tableView} return ( - <> - {buttonElement && ( - - Install Data Formulator locally to use database.
    - Link: e.stopPropagation()} - > - https://github.com/microsoft/data-formulator - - - ) : ""} - placement="top" - > - - - -     + + {mainContent} + {isUploading && ( + + + )} - {setTableDialogOpen(false)}} - open={tableDialogOpen} - sx={{ '& .MuiDialog-paper': { maxWidth: '100%', maxHeight: 800, minWidth: 800 } }} - > - - Database - setTableDialogOpen(false)} - > - - - - - {mainContent} - {isUploading && ( - - - - )} - - - + + ); } @@ -1043,129 +1154,366 @@ export const DataLoaderForm: React.FC<{ const theme = useTheme(); const params = useSelector((state: DataFormulatorState) => state.dataLoaderConnectParams[dataLoaderType] ?? {}); - const [tableMetadata, setTableMetadata] = useState>({}); let [displaySamples, setDisplaySamples] = useState>({}); + const [tableMetadata, setTableMetadata] = useState>({}); + let [displaySamples, setDisplaySamples] = useState>({}); let [tableFilter, setTableFilter] = useState(""); - const [selectedTables, setSelectedTables] = useState>(new Set()); - - const [displayAuthInstructions, setDisplayAuthInstructions] = useState(false); + const [tableImportConfigs, setTableImportConfigs] = useState>({}); + const [subsetConfigAnchor, setSubsetConfigAnchor] = useState<{element: HTMLElement, tableName: string} | null>(null); + + // Helper to get import config for a table (defaults to 'none') + const getTableConfig = (tableName: string): TableImportConfig => { + return tableImportConfigs[tableName] ?? { mode: 'none' }; + }; + + // Helper to update config for a specific table + const updateTableConfig = (tableName: string, config: TableImportConfig) => { + setTableImportConfigs(prev => ({ ...prev, [tableName]: config })); + }; + + // Get selected tables (those with mode !== 'none') + const selectedTables = Object.entries(tableImportConfigs) + .filter(([_, config]) => config.mode !== 'none') + .map(([tableName, _]) => tableName); let [isConnecting, setIsConnecting] = useState(false); - let [mode, setMode] = useState<"view tables" | "query">("view tables"); const toggleDisplaySamples = (tableName: string) => { setDisplaySamples({...displaySamples, [tableName]: !displaySamples[tableName]}); } - const handleModeChange = (event: React.MouseEvent, newMode: "view tables" | "query") => { - if (newMode != null) { - setMode(newMode); - } - }; - let tableMetadataBox = [ - - - View Tables - Query Data - - - , - mode === "view tables" && + - - {Object.entries(tableMetadata).map(([tableName, metadata]) => { - return [ - - - toggleDisplaySamples(tableName)}> - {displaySamples[tableName] ? : } - - - - {tableName} - ({metadata.row_count > 0 ? `${metadata.row_count} rows × ` : ""}{metadata.columns.length} cols) - - - - {metadata.columns.map((column: any) => ( - - ))} - - - + + + Table Name + Columns + Import Options + + + + {Object.entries(tableMetadata).map(([tableName, metadata]) => { + return [ + + + { + e.stopPropagation(); + toggleDisplaySamples(tableName); + }}> + {displaySamples[tableName] ? : } + + + + {tableName} + ({metadata.row_count > 0 ? `${metadata.row_count} rows × ` : ""}{metadata.columns.length} cols) + + + + {metadata.columns.map((column: any) => ( + + ))} + + + + + { + if (newMode === null) return; // Prevent deselecting all + if (newMode === 'none') { + updateTableConfig(tableName, { mode: 'none' }); + } else if (newMode === 'full') { + updateTableConfig(tableName, { mode: 'full' }); + } else if (newMode === 'subset') { + // Initialize with default values + updateTableConfig(tableName, { + mode: 'subset', + rowLimit: Math.min(1000, metadata.row_count || 1000), + sortColumns: [], + sortOrder: 'asc' + }); + } + }} + > + + + Skip + + + + + Full + + + { + e.stopPropagation(); + setSubsetConfigAnchor({ element: e.currentTarget, tableName }); + }} sx={{ + px: 1, py: 0, fontSize: 11, textTransform: 'none', + '&.Mui-selected': { backgroundColor: '#f9a825', color: 'white' }, + '&.Mui-selected:hover': { backgroundColor: '#f57f17' } + }}> + + Subset + + + + + {getTableConfig(tableName).mode === 'subset' && ( + + + + )} + + + , + + + + + { + return Object.fromEntries(Object.entries(row).map(([key, value]: [string, any]) => { + return [key, String(value)]; + })); + })} + columnDefs={metadata.columns.map((column: any) => ({id: column.name, label: column.name}))} + rowsPerPageNum={-1} + compact={false} + isIncompleteTable={metadata.row_count > 10} + /> + + + + ] + })} + +
    +     , + // Subset configuration popover + setSubsetConfigAnchor(null)} + anchorOrigin={{ + vertical: 'bottom', + horizontal: 'left', + }} + transformOrigin={{ + vertical: 'top', + horizontal: 'left', + }} + > + {subsetConfigAnchor && (() => { + const tableName = subsetConfigAnchor.tableName; + const config = getTableConfig(tableName); + const metadata = tableMetadata[tableName]; + if (config.mode !== 'subset' || !metadata) return null; + + return ( + + + Create a subset of "{tableName}" + + + Row Limit (max: {metadata.row_count || 'unknown'} rows) + + + { - const newSelected = new Set(selectedTables); - if (e.target.checked) { - newSelected.add(tableName); - } else { - newSelected.delete(tableName); + const value = parseInt(e.target.value) || 1; + const maxRows = metadata.row_count || 100000; + updateTableConfig(tableName, { + ...config, + rowLimit: Math.min(Math.max(1, value), maxRows) + }); + }} + slotProps={{ + input: { + inputProps: { + min: 1, + max: metadata.row_count || 100000, + step: 100 + } } - setSelectedTables(newSelected); }} + fullWidth + sx={{ mb: 1, '& .MuiInputBase-root': { fontSize: 12 } }} /> - - , - - - - - { - return Object.fromEntries(Object.entries(row).map(([key, value]: [string, any]) => { - return [key, String(value)]; - })); - })} - columnDefs={metadata.columns.map((column: any) => ({id: column.name, label: column.name}))} - rowsPerPageNum={-1} - compact={false} - isIncompleteTable={metadata.row_count > 10} - /> - - - - ] - })} - - -     , - mode === "view tables" && Object.keys(tableMetadata).length > 0 && + { + updateTableConfig(tableName, { + ...config, + rowLimit: value as number + }); + }} + min={1} + max={metadata.row_count || 10000} + step={Math.max(1, Math.floor((metadata.row_count || 10000) / 100))} + valueLabelDisplay="auto" + /> + + + + Sort By (optional) + + + col.name)} + value={config.sortColumns} + onChange={(_, newValue) => { + updateTableConfig(tableName, { + ...config, + sortColumns: newValue + }); + }} + renderInput={(params) => ( + + )} + renderTags={(value, getTagProps) => + value.map((option, index) => ( + + )) + } + /> + {config.sortColumns.length > 0 && ( + + { + if (newValue) { + updateTableConfig(tableName, { + ...config, + sortOrder: newValue + }); + } + }} + size="small" + sx={{ height: 24 }} + > + + ↑ Asc + + + ↓ Desc + + + + )} + + + + + +     + ); + })()} + , + Object.keys(tableMetadata).length > 0 && - , - mode === "query" && ({name: t, fields: tableMetadata[t].columns.map((c: any) => c.name)}))} - dataLoaderParams={params} onImport={onImport} onFinish={onFinish} /> +     ] + const isConnected = Object.keys(tableMetadata).length > 0; + return ( + + Import tables from {dataLoaderType} + {isConnecting && } - - Data Connector ({dataLoaderType}) - - - {paramDefs.map((paramDef) => ( - - 0} - sx={{width: "270px", - '& .MuiInputLabel-root': {fontSize: 14}, - '& .MuiInputBase-root': {fontSize: 14}, - '& .MuiInputBase-input::placeholder': {fontSize: 12, fontStyle: "italic"} - }} - variant="standard" - size="small" - required={paramDef.required} - key={paramDef.name} - label={paramDef.name} - value={params[paramDef.name] ?? ''} - placeholder={paramDef.default ? `e.g. ${paramDef.default}` : paramDef.description} - onChange={(event: any) => { - dispatch(dfActions.updateDataLoaderConnectParam({ - dataLoaderType, paramName: paramDef.name, - paramValue: event.target.value})); - }} - slotProps={{ - inputLabel: {shrink: true} - }} - /> + {isConnected ? ( + // Connected state: show connection parameters and disconnect button + + + + + {paramDefs.filter((paramDef) => params[paramDef.name]).map((paramDef, index) => ( + + + {paramDef.name}: + + + {params[paramDef.name] || '(empty)'} + + {index < paramDefs.filter((paramDef) => params[paramDef.name]).length - 1 && ( + + • + + )} + + ))} + + + + + + + + table filter + + + setTableFilter(event.target.value)} + /> + + + + + + - ))} - - - table filter - } - placeholder="load only tables containing keywords" - value={tableFilter} - onChange={(event) => setTableFilter(event.target.value)} - slotProps={{ - inputLabel: {shrink: true}, - }} - /> - {paramDefs.length > 0 && - - - } - - - - - - - { - - + .catch((error: any) => { + onFinish("error", `Failed to fetch data loader tables, please check the server is running`); + setIsConnecting(false); + }); + }}> + Connect {tableFilter.trim() ? "with filter" : ""} + } + + + + + {authInstructions.trim()} - - - } - - {Object.keys(tableMetadata).length > 0 && tableMetadataBox } + + )} ); -} - -export const DataQueryForm: React.FC<{ - dataLoaderType: string, - availableTables: {name: string, fields: string[]}[], - dataLoaderParams: Record, - onImport: () => void, - onFinish: (status: "success" | "error", message: string) => void -}> = ({dataLoaderType, availableTables, dataLoaderParams, onImport, onFinish}) => { - - let activeModel = useSelector(dfSelectors.getActiveModel); - - const [selectedTables, setSelectedTables] = useState(availableTables.map(t => t.name).slice(0, 5)); - - const [waiting, setWaiting] = useState(false); - - const [query, setQuery] = useState("-- query the data source / describe your goal and ask AI to help you write the query\n"); - const [queryResult, setQueryResult] = useState<{ - status: string, - message: string, - sample: any[], - code: string, - } | undefined>(undefined); - const [queryResultName, setQueryResultName] = useState(""); - - const aiCompleteQuery = (query: string) => { - if (queryResult?.status === "error") { - setQueryResult(undefined); - } - let data = { - data_source_metadata: { - data_loader_type: dataLoaderType, - tables: availableTables.filter(t => selectedTables.includes(t.name)) - }, - query: query, - model: activeModel - } - setWaiting(true); - fetch(getUrls().QUERY_COMPLETION, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify(data) - }) - .then(response => response.json()) - .then(data => { - setWaiting(false); - if (data.status === "ok") { - setQuery(data.query); - } else { - onFinish("error", data.reasoning); - } - }) - .catch(error => { - setWaiting(false); - onFinish("error", `Failed to complete query please try again.`); - }); - } - - const handleViewQuerySample = (query: string) => { - setQueryResult(undefined); - setWaiting(true); - fetch(getUrls().DATA_LOADER_VIEW_QUERY_SAMPLE, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - data_loader_type: dataLoaderType, - data_loader_params: dataLoaderParams, - query: query - }) - }) - .then(response => response.json()) - .then(data => { - setWaiting(false); - if (data.status === "success") { - setQueryResult({ - status: "success", - message: "Data loaded successfully", - sample: data.sample, - code: query - }); - let newName = `r_${Math.random().toString(36).substring(2, 4)}`; - setQueryResultName(newName); - } else { - setQueryResult({ - status: "error", - message: data.message, - sample: [], - code: query - }); - } - }) - .catch(error => { - setWaiting(false); - setQueryResult({ - status: "error", - message: `Failed to view query sample, please try again.`, - sample: [], - code: query - }); - }); - } - - const handleImportQueryResult = () => { - setWaiting(true); - fetch(getUrls().DATA_LOADER_INGEST_DATA_FROM_QUERY, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - data_loader_type: dataLoaderType, - data_loader_params: dataLoaderParams, - query: queryResult?.code ?? query, - name_as: queryResultName - }) - }) - .then(response => response.json()) - .then(data => { - setWaiting(false); - if (data.status === "success") { - onFinish("success", "Data imported successfully"); - } else { - onFinish("error", data.reasoning); - } - }) - .catch(error => { - setWaiting(false); - onFinish("error", `Failed to import data, please try again.`); - }); - } - - let queryResultBox = queryResult?.status === "success" && queryResult.sample.length > 0 ? [ - - ({id: t, label: t}))} rowsPerPageNum={-1} compact={false} /> - , - - - setQueryResultName(event.target.value)} - /> - - - ] : []; - - return ( - - {waiting && - - } - - - query from tables: - - {availableTables.map((table) => ( - : undefined} - color={selectedTables.includes(table.name) ? "primary" : "default"} variant="outlined" - sx={{ fontSize: 11, margin: 0.25, - height: 20, borderRadius: 0.5, - borderColor: selectedTables.includes(table.name) ? "primary.main" : "rgba(0, 0, 0, 0.1)", - color: selectedTables.includes(table.name) ? "primary.main" : "text.secondary", - '&:hover': { - backgroundColor: "rgba(0, 0, 0, 0.07)", - } - }} - size="small" - onClick={() => { - setSelectedTables(selectedTables.includes(table.name) ? selectedTables.filter(t => t !== table.name) : [...selectedTables, table.name]); - }} - /> - ))} - - - - { - setQuery(tempCode); - }} - highlight={code => Prism.highlight(code, Prism.languages.sql, 'sql')} - padding={10} - style={{ - minHeight: queryResult ? 60 : 200, - fontFamily: '"Fira code", "Fira Mono", monospace', - fontSize: 12, - paddingBottom: '24px', - backgroundColor: "rgba(0, 0, 0, 0.03)", - overflowY: "auto" - }} - /> - - {queryResult?.status === "error" && - - {queryResult?.message} - - } - - - {queryResult?.status === "error" && } - - - {queryResult && queryResultBox} - - - ) } \ No newline at end of file diff --git a/src/views/DataFormulator.tsx b/src/views/DataFormulator.tsx index ead4e941..f9cea9d5 100644 --- a/src/views/DataFormulator.tsx +++ b/src/views/DataFormulator.tsx @@ -18,7 +18,6 @@ import { Allotment } from "allotment"; import "allotment/dist/style.css"; import { - Typography, Box, Tooltip, @@ -27,13 +26,6 @@ import { useTheme, alpha, } from '@mui/material'; -import { - FolderOpen as FolderOpenIcon, - ContentPaste as ContentPasteIcon, - Category as CategoryIcon, - CloudQueue as CloudQueueIcon, - AutoFixNormal as AutoFixNormalIcon, -} from '@mui/icons-material'; import { FreeDataViewFC } from './DataView'; import { VisualizationViewFC } from './VisualizationView'; @@ -41,19 +33,17 @@ import { VisualizationViewFC } from './VisualizationView'; import { ConceptShelf } from './ConceptShelf'; import { DndProvider } from 'react-dnd' import { HTML5Backend } from 'react-dnd-html5-backend' -import { TableCopyDialogV2, DatasetSelectionDialog } from './TableSelectionView'; -import { TableUploadDialog } from './TableSelectionView'; import { toolName } from '../app/App'; import { DataThread } from './DataThread'; import dfLogo from '../assets/df-logo.png'; import exampleImageTable from "../assets/example-image-table.png"; import { ModelSelectionButton } from './ModelSelectionDialog'; -import { DBTableSelectionDialog } from './DBTableManager'; import { getUrls } from '../app/utils'; -import { DataLoadingChatDialog } from './DataLoadingChat'; +import { UnifiedDataUploadDialog, UploadTabType, DataLoadMenu } from './UnifiedDataUploadDialog'; import { ReportView } from './ReportView'; import { ExampleSession, exampleSessions, ExampleSessionCard } from './ExampleSessions'; +import { useDataRefresh, useDerivedTableRefresh } from '../app/useDataRefresh'; export const DataFormulatorFC = ({ }) => { @@ -61,9 +51,22 @@ export const DataFormulatorFC = ({ }) => { const models = useSelector((state: DataFormulatorState) => state.models); const selectedModelId = useSelector((state: DataFormulatorState) => state.selectedModelId); const viewMode = useSelector((state: DataFormulatorState) => state.viewMode); + const serverConfig = useSelector((state: DataFormulatorState) => state.serverConfig); const theme = useTheme(); const dispatch = useDispatch(); + + // Set up automatic refresh of derived tables when source data changes + useDerivedTableRefresh(); + + // State for unified data upload dialog + const [uploadDialogOpen, setUploadDialogOpen] = useState(false); + const [uploadDialogInitialTab, setUploadDialogInitialTab] = useState('menu'); + + const openUploadDialog = (tab: UploadTabType) => { + setUploadDialogInitialTab(tab); + setUploadDialogOpen(true); + }; const handleLoadExampleSession = (session: ExampleSession) => { dispatch(dfActions.addMessages({ @@ -224,7 +227,7 @@ export const DataFormulatorFC = ({ }) => { {viewMode === 'editor' ? ( <> {visPane} - + {/* */} ) : ( @@ -273,20 +276,16 @@ export const DataFormulatorFC = ({ }) => { Explore data with visualizations, powered by AI agents. - - To begin, - extract}/>{' '} - data from images or text documents, load {' '} - examples}/>, - upload data from{' '} - clipboard} disabled={false}/> or {' '} - files} disabled={false}/>, - - or connect to a{' '} - database}/>. - + openUploadDialog(tab)} + serverConfig={serverConfig} + variant="page" + /> + setUploadDialogOpen(false)} + initialTab={uploadDialogInitialTab} + /> @@ -331,7 +330,7 @@ export const DataFormulatorFC = ({ }) => { zIndex: 1000, }}> - + {toolName} diff --git a/src/views/DataLoadingChat.tsx b/src/views/DataLoadingChat.tsx index a3273a55..66021496 100644 --- a/src/views/DataLoadingChat.tsx +++ b/src/views/DataLoadingChat.tsx @@ -4,9 +4,7 @@ import * as React from 'react'; import { useEffect, useRef, useState } from 'react'; -import { Box, Button, Divider, IconButton, Typography, Dialog, DialogTitle, DialogContent, Tooltip, CircularProgress } from '@mui/material'; -import RestartAltIcon from '@mui/icons-material/RestartAlt'; -import CloseIcon from '@mui/icons-material/Close'; +import { Box, Button, Divider, IconButton, Typography, Tooltip, CircularProgress, alpha, useTheme } from '@mui/material'; import { useDispatch, useSelector } from 'react-redux'; @@ -32,7 +30,7 @@ const getUniqueTableName = (baseName: string, existingNames: Set): strin }; export const DataLoadingChat: React.FC = () => { - + const theme = useTheme(); const dispatch = useDispatch(); const inputBoxRef = useRef<(() => void) | null>(null); const abortControllerRef = useRef(null); @@ -89,12 +87,6 @@ export const DataLoadingChat: React.FC = () => { } }; - if (!existOutputBlocks && !streamingContent) { - return - - - } - const thinkingBanner = ( { ); - - let chatCard = ( - - - {/* Left: Chat panel */} - + - - {threadsComponent} + gap: 2, + }}> + - + ); + } - - - {streamingContent && ( - - {thinkingBanner} - - {streamingContent.trim()} - - - )} - - {/* Right: Data preview panel */} - {(existOutputBlocks && !streamingContent) && ( - + + {/* Left sidebar - Thread list (similar to DBTablePane) */} + + - - - {selectedTable && ( - - {selectedTable?.name} - - )} - - - - {selectedTable ? ( - + overflowY: 'auto', + overflowX: 'hidden', + flex: 1, + minHeight: 0, + height: '100%', + position: 'relative', + overscrollBehavior: 'contain', + px: 0.5, + pt: 1 + }}> + {threadsComponent.length > 0 ? ( + threadsComponent ) : ( - - No data available + + No extraction threads yet )} + + + + + - {/* Bottom submit bar */} - - - + {selectedTable ? ( + + + + ) : ( + + + Select a table from the left to preview + + + )} + + {/* Bottom submit bar */} + {selectedTable && ( + + + + + )} + - + ) : null} - )} + ); - - return chatCard; -}; - -export interface DataLoadingChatDialogProps { - buttonElement?: any; - disabled?: boolean; - onOpen?: () => void; - // Controlled mode props - open?: boolean; - onClose?: () => void; -} - -export const DataLoadingChatDialog: React.FC = ({ - buttonElement, - disabled = false, - onOpen, - open: controlledOpen, - onClose, -}) => { - const [internalOpen, setInternalOpen] = useState(false); - const dispatch = useDispatch(); - const dataCleanBlocks = useSelector((state: DataFormulatorState) => state.dataCleanBlocks); - - // Support both controlled and uncontrolled modes - const isControlled = controlledOpen !== undefined; - const dialogOpen = isControlled ? controlledOpen : internalOpen; - const setDialogOpen = isControlled - ? (open: boolean) => { if (!open && onClose) onClose(); } - : setInternalOpen; - - return ( - <> - {buttonElement && ( - - )} - setDialogOpen(false)} - open={dialogOpen} - sx={{ '& .MuiDialog-paper': { maxWidth: '100%', maxHeight: 840, minWidth: 800 } }} - > - - Extract Data - {dataCleanBlocks.length > 0 && - { - dispatch(dfActions.resetDataCleanBlocks()); - }}> - - - } - setDialogOpen(false)} - aria-label="close" - > - - - - - - - - - ); }; - diff --git a/src/views/DataThread.tsx b/src/views/DataThread.tsx index d873edbb..3cffa9a4 100644 --- a/src/views/DataThread.tsx +++ b/src/views/DataThread.tsx @@ -22,7 +22,11 @@ import { Popper, Paper, ClickAwayListener, - Badge + Badge, + Menu, + MenuItem, + Switch, + FormControlLabel, } from '@mui/material'; import { VegaLite } from 'react-vega' @@ -61,10 +65,18 @@ import ChevronLeftIcon from '@mui/icons-material/ChevronLeft'; import ChevronRightIcon from '@mui/icons-material/ChevronRight'; import CloudQueueIcon from '@mui/icons-material/CloudQueue'; import AttachFileIcon from '@mui/icons-material/AttachFile'; +import MoreVertIcon from '@mui/icons-material/MoreVert'; +import RefreshIcon from '@mui/icons-material/Refresh'; +import StreamIcon from '@mui/icons-material/Stream'; import { alpha } from '@mui/material/styles'; import { dfSelectors } from '../app/dfSlice'; +import { RefreshDataDialog } from './RefreshDataDialog'; +import { getUrls } from '../app/utils'; +import { AppDispatch } from '../app/store'; +import StopIcon from '@mui/icons-material/Stop'; +import { useDataRefresh } from '../app/useDataRefresh'; export const ThinkingBanner = (message: string, sx?: SxProps) => ( ( ); +// Streaming Settings Popup Component +const StreamingSettingsPopup = memo<{ + open: boolean; + anchorEl: HTMLElement | null; + onClose: () => void; + table: DictTable; + onUpdateSettings: (autoRefresh: boolean, refreshIntervalSeconds?: number) => void; + onRefreshNow?: () => void; +}>(({ open, anchorEl, onClose, table, onUpdateSettings, onRefreshNow }) => { + const [refreshInterval, setRefreshInterval] = useState( + table.source?.refreshIntervalSeconds || 60 + ); + const [autoRefresh, setAutoRefresh] = useState( + table.source?.autoRefresh || false + ); + const [selectMenuOpen, setSelectMenuOpen] = useState(false); + const [isRefreshing, setIsRefreshing] = useState(false); + + useEffect(() => { + if (open) { + setRefreshInterval(table.source?.refreshIntervalSeconds || 60); + setAutoRefresh(table.source?.autoRefresh || false); + } + }, [open, table.source]); + + const handleAutoRefreshChange = (enabled: boolean) => { + setAutoRefresh(enabled); + onUpdateSettings(enabled, enabled ? refreshInterval : undefined); + if (!enabled) { + onClose(); + } + }; + + const handleIntervalChange = (interval: number) => { + setRefreshInterval(interval); + if (autoRefresh) { + onUpdateSettings(true, interval); + } + }; + + const handleRefreshNow = async () => { + if (onRefreshNow && !isRefreshing) { + setIsRefreshing(true); + try { + await onRefreshNow(); + } finally { + setIsRefreshing(false); + } + } + }; + + const handleClickAway = (event: MouseEvent | TouchEvent) => { + // Don't close if the select menu is open + if (selectMenuOpen) { + return; + } + // Don't close if clicking on the select menu or menu items + const target = event.target as HTMLElement; + if ( + target.closest('.MuiMenu-root') || + target.closest('.MuiPaper-root')?.classList.contains('MuiMenu-paper') || + target.closest('[role="menuitem"]') || + target.closest('[role="listbox"]') + ) { + return; + } + onClose(); + }; + + return ( + + + + + + handleAutoRefreshChange(e.target.checked)} + size="small" + /> + } + label={ + + Watch for updates + + } + sx={{ mr: 0 }} + /> + {autoRefresh && ( + + + every + + handleIntervalChange(Number(e.target.value))} + slotProps={{ + select: { + open: selectMenuOpen, + onOpen: () => setSelectMenuOpen(true), + onClose: () => setSelectMenuOpen(false) + } + }} + sx={{ + minWidth: 70, + '& .MuiInputBase-root': { fontSize: 11, height: 28 }, + '& .MuiSelect-select': { py: 0.5 } + }} + > + 1s + 10s + 30s + 1m + 5m + 10m + 30m + 1h + 24h + + + )} + {onRefreshNow && ( + + )} + + + + + + ); +}); + // Metadata Popup Component const MetadataPopup = memo<{ open: boolean; @@ -458,12 +634,62 @@ const EditableTableName: FC<{ ); }; +// Compact view for thread0 - displays table cards with charts in a simple grid +// Reuses SingleThreadGroupView with compact mode +let CompactThread0View: FC<{ + scrollRef: any, + leafTables: DictTable[]; + chartElements: { tableId: string, chartId: string, element: any }[]; + sx?: SxProps +}> = function ({ + scrollRef, + leafTables, + chartElements, + sx +}) { + const theme = useTheme(); + + return ( + + + + + workspace + + + + + + + + ); +} + let SingleThreadGroupView: FC<{ scrollRef: any, threadIdx: number, leafTables: DictTable[]; chartElements: { tableId: string, chartId: string, element: any }[]; usedIntermediateTableIds: string[], + compact?: boolean, // When true, only show table cards in a simple column (for thread0) sx?: SxProps }> = function ({ scrollRef, @@ -471,10 +697,12 @@ let SingleThreadGroupView: FC<{ leafTables, chartElements, usedIntermediateTableIds, // tables that have been used + compact = false, sx }) { let tables = useSelector((state: DataFormulatorState) => state.tables); + const { manualRefresh } = useDataRefresh(); let leafTableIds = leafTables.map(lt => lt.id); let parentTableId = leafTables[0].derive?.trigger.tableId || undefined; @@ -490,6 +718,19 @@ let SingleThreadGroupView: FC<{ const [selectedTableForMetadata, setSelectedTableForMetadata] = useState(null); const [metadataAnchorEl, setMetadataAnchorEl] = useState(null); + // Table menu state + const [tableMenuAnchorEl, setTableMenuAnchorEl] = useState(null); + const [selectedTableForMenu, setSelectedTableForMenu] = useState(null); + + // Refresh data dialog state + const [refreshDialogOpen, setRefreshDialogOpen] = useState(false); + const [selectedTableForRefresh, setSelectedTableForRefresh] = useState(null); + const [isRefreshing, setIsRefreshing] = useState(false); + + // Streaming settings popup state + const [streamingSettingsPopupOpen, setStreamingSettingsPopupOpen] = useState(false); + const [selectedTableForStreamingSettings, setSelectedTableForStreamingSettings] = useState(null); + const [streamingSettingsAnchorEl, setStreamingSettingsAnchorEl] = useState(null); let handleUpdateTableDisplayId = (tableId: string, displayId: string) => { dispatch(dfActions.updateTableDisplayId({ @@ -519,6 +760,150 @@ let SingleThreadGroupView: FC<{ } }; + // Table menu handlers + const handleOpenTableMenu = (table: DictTable, anchorEl: HTMLElement) => { + setSelectedTableForMenu(table); + setTableMenuAnchorEl(anchorEl); + }; + + const handleCloseTableMenu = () => { + setTableMenuAnchorEl(null); + setSelectedTableForMenu(null); + }; + + // Refresh data handlers + const handleOpenRefreshDialog = (table: DictTable) => { + setSelectedTableForRefresh(table); + setRefreshDialogOpen(true); + handleCloseTableMenu(); + }; + + const handleCloseRefreshDialog = () => { + setRefreshDialogOpen(false); + setSelectedTableForRefresh(null); + }; + + // Streaming settings handlers + const handleOpenStreamingSettingsPopup = (table: DictTable, anchorEl: HTMLElement) => { + setSelectedTableForStreamingSettings(table); + setStreamingSettingsAnchorEl(anchorEl); + setStreamingSettingsPopupOpen(true); + }; + + const handleCloseStreamingSettingsPopup = () => { + setStreamingSettingsPopupOpen(false); + setSelectedTableForStreamingSettings(null); + setStreamingSettingsAnchorEl(null); + }; + + const handleUpdateStreamingSettings = (autoRefresh: boolean, refreshIntervalSeconds?: number) => { + if (selectedTableForStreamingSettings) { + dispatch(dfActions.updateTableSourceRefreshSettings({ + tableId: selectedTableForStreamingSettings.id, + autoRefresh, + refreshIntervalSeconds + })); + } + }; + + // Function to refresh derived tables + const refreshDerivedTables = async (sourceTableId: string, newRows: any[]) => { + // Find all tables that are derived from this source table + const derivedTables = tables.filter(t => t.derive?.source?.includes(sourceTableId)); + + for (const derivedTable of derivedTables) { + if (derivedTable.derive && derivedTable.derive.code) { + // Gather all parent tables for this derived table + const parentTableData = derivedTable.derive.source.map(sourceId => { + const sourceTable = tables.find(t => t.id === sourceId); + if (sourceTable) { + // Use the new rows if this is the table being refreshed + const rows = sourceId === sourceTableId ? newRows : sourceTable.rows; + return { + name: sourceTable.id, + rows: rows + }; + } + return null; + }).filter(t => t !== null); + + if (parentTableData.length > 0) { + try { + const response = await fetch(getUrls().REFRESH_DERIVED_DATA, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + input_tables: parentTableData, + code: derivedTable.derive.code + }) + }); + + const result = await response.json(); + if (result.status === 'ok' && result.rows) { + // Update the derived table with new rows + dispatch(dfActions.updateTableRows({ + tableId: derivedTable.id, + rows: result.rows + })); + + // Recursively refresh tables derived from this one + await refreshDerivedTables(derivedTable.id, result.rows); + } else { + console.error(`Failed to refresh derived table ${derivedTable.id}:`, result.message); + dispatch(dfActions.addMessages({ + timestamp: Date.now(), + type: 'error', + component: 'data refresh', + value: `Failed to refresh derived table "${derivedTable.displayId || derivedTable.id}": ${result.message || 'Unknown error'}` + })); + } + } catch (error) { + console.error(`Error refreshing derived table ${derivedTable.id}:`, error); + dispatch(dfActions.addMessages({ + timestamp: Date.now(), + type: 'error', + component: 'data refresh', + value: `Error refreshing derived table "${derivedTable.displayId || derivedTable.id}"` + })); + } + } + } + } + }; + + const handleRefreshComplete = async (newRows: any[]) => { + if (!selectedTableForRefresh) return; + + setIsRefreshing(true); + try { + // Update the source table with new rows + dispatch(dfActions.updateTableRows({ + tableId: selectedTableForRefresh.id, + rows: newRows + })); + + // Refresh all derived tables + await refreshDerivedTables(selectedTableForRefresh.id, newRows); + + dispatch(dfActions.addMessages({ + timestamp: Date.now(), + type: 'success', + component: 'data refresh', + value: `Successfully refreshed data for "${selectedTableForRefresh.displayId || selectedTableForRefresh.id}" and updated derived tables.` + })); + } catch (error) { + console.error('Error during refresh:', error); + dispatch(dfActions.addMessages({ + timestamp: Date.now(), + type: 'error', + component: 'data refresh', + value: `Error refreshing data: ${error}` + })); + } finally { + setIsRefreshing(false); + } + }; + let buildTriggerCard = (trigger: Trigger) => { let selectedClassName = trigger.chart?.id == focusedChartId ? 'selected-card' : ''; @@ -543,7 +928,7 @@ let SingleThreadGroupView: FC<{ ; } - let buildTableCard = (tableId: string) => { + let buildTableCard = (tableId: string, compact = false) => { if (parentTable && tableId == parentTable.id && parentTable.anchored && tableIdList.length > 1) { let table = tables.find(t => t.id == tableId); @@ -609,13 +994,22 @@ let SingleThreadGroupView: FC<{ // only charts without dependency can be deleted let tableDeleteEnabled = !tables.some(t => t.derive?.trigger.tableId == tableId); - let tableCardIcon = ( table?.anchored ? - + ) : ( + : - ) + color: iconColor, + opacity: iconOpacity, + }} /> + ) let regularTableBox = c.chartId == focusedChartId) ? scrollRef : null} sx={{ padding: '0px' }}> @@ -637,27 +1031,83 @@ let SingleThreadGroupView: FC<{ }}> - t.derive?.trigger.tableId == tableId)} - onClick={(event) => { - event.stopPropagation(); - dispatch(dfActions.updateTableAnchored({tableId: tableId, anchored: !table?.anchored})); - }}> - {tableCardIcon} - + {/* For non-derived tables: icon opens menu; for derived tables: icon toggles anchored */} + {table?.derive == undefined ? ( + + { + event.stopPropagation(); + handleOpenTableMenu(table!, event.currentTarget); + }}> + {tableCardIcon} + + + ) : ( + t.derive?.trigger.tableId == tableId)} + onClick={(event) => { + event.stopPropagation(); + dispatch(dfActions.updateTableAnchored({tableId: tableId, anchored: !table?.anchored})); + }}> + {tableCardIcon} + + )} - {table?.virtual? : ""} + {/* Only show streaming icon when actively watching for updates */} + {(table?.source?.type === 'stream' || table?.source?.type === 'database') && table?.source?.autoRefresh ? ( + + { + event.stopPropagation(); + handleOpenStreamingSettingsPopup(table!, event.currentTarget); + }} + sx={{ + padding: 0.25, + '&:hover': { + transform: 'scale(1.2)', + transition: 'all 0.1s linear' + } + }} + > + + + + ) : ""} {focusedTableId == tableId ? {table?.displayId || tableId}} - {table?.derive == undefined && - { - event.stopPropagation(); - handleOpenMetadataPopup(table!, event.currentTarget); - }} - > - - - } - - {tableDeleteEnabled && - { - event.stopPropagation(); - dispatch(dfActions.deleteTable(tableId)); - }} - > - - - } - - + + {/* Delete button - shown for all deletable tables */} + {tableDeleteEnabled && ( + + { + event.stopPropagation(); + dispatch(dfActions.deleteTable(tableId)); + }} + > + + + + )} @@ -756,7 +1188,7 @@ let SingleThreadGroupView: FC<{ backgroundSize: '1px 6px, 3px 100%' }}> } - + {releventChartElements} {agentActionBox} @@ -829,6 +1261,111 @@ let SingleThreadGroupView: FC<{ ; }); + // Compact mode: just show leaf table cards in a simple column + if (compact) { + // For compact mode, ensure highlightedTableIds includes focused table if it's a leaf + if (focusedTableId && leafTableIds.includes(focusedTableId)) { + highlightedTableIds = [focusedTableId]; + } + + return ( + + {leafTables.map((table) => { + const tableCardResult = buildTableCard(table.id, compact); + // buildTableCard returns an array [regularTableBox, chartBox] + // In compact mode, we want to show them stacked + return ( + + {tableCardResult} + + ); + })} + + e.stopPropagation()} + > + { + e.stopPropagation(); + if (selectedTableForMenu) { + handleOpenMetadataPopup(selectedTableForMenu, tableMenuAnchorEl!); + } + handleCloseTableMenu(); + }} + sx={{ fontSize: '12px', display: 'flex', alignItems: 'center', gap: 1 }} + > + + {selectedTableForMenu?.attachedMetadata ? "Edit metadata" : "Attach metadata"} + + {/* Watch for updates option - only shown when table has stream/database source but not actively watching */} + {selectedTableForMenu && + (selectedTableForMenu.source?.type === 'stream' || selectedTableForMenu.source?.type === 'database') && + ( + { + e.stopPropagation(); + if (selectedTableForMenu) { + handleOpenStreamingSettingsPopup(selectedTableForMenu, tableMenuAnchorEl!); + } + handleCloseTableMenu(); + }} + sx={{ fontSize: '12px', display: 'flex', alignItems: 'center', gap: 1 }} + > + + Watch for updates + + )} + {/* Refresh data - hidden for database tables */} + {selectedTableForMenu?.source?.type !== 'database' && ( + { + e.stopPropagation(); + if (selectedTableForMenu) { + handleOpenRefreshDialog(selectedTableForMenu); + } + }} + sx={{ fontSize: '12px', display: 'flex', alignItems: 'center', gap: 1 }} + > + + Refresh data + + )} + + {selectedTableForRefresh && ( + + )} + {selectedTableForStreamingSettings && ( + manualRefresh(selectedTableForStreamingSettings.id)} + /> + )} + + ); + } + return - {`thread - ${threadIdx + 1}`} + {threadIdx === -1 ? 'thread0' : `thread - ${threadIdx + 1}`} @@ -888,8 +1425,102 @@ let SingleThreadGroupView: FC<{ initialValue={selectedTableForMetadata?.attachedMetadata || ''} tableName={selectedTableForMetadata?.displayId || selectedTableForMetadata?.id || ''} /> + + {/* Table actions menu for non-derived, non-virtual tables */} + e.stopPropagation()} + > + { + e.stopPropagation(); + if (selectedTableForMenu) { + handleOpenMetadataPopup(selectedTableForMenu, tableMenuAnchorEl!); + } + handleCloseTableMenu(); + }} + sx={{ fontSize: '12px', display: 'flex', alignItems: 'center', gap: 1 }} + > + + {selectedTableForMenu?.attachedMetadata ? "Edit metadata" : "Attach metadata"} + + {/* Watch for updates option - only shown when table has stream/database source but not actively watching */} + {selectedTableForMenu && + (selectedTableForMenu.source?.type === 'stream' || selectedTableForMenu.source?.type === 'database') && + !selectedTableForMenu.source?.autoRefresh && ( + { + e.stopPropagation(); + if (selectedTableForMenu) { + handleOpenStreamingSettingsPopup(selectedTableForMenu, tableMenuAnchorEl!); + } + handleCloseTableMenu(); + }} + sx={{ fontSize: '12px', display: 'flex', alignItems: 'center', gap: 1 }} + > + + Watch for updates + + )} + {/* Refresh data - hidden for database tables */} + {selectedTableForMenu?.source?.type !== 'database' && ( + { + e.stopPropagation(); + if (selectedTableForMenu) { + handleOpenRefreshDialog(selectedTableForMenu); + } + }} + sx={{ fontSize: '12px', display: 'flex', alignItems: 'center', gap: 1 }} + > + + Refresh data + + )} + { + e.stopPropagation(); + if (selectedTableForMenu) { + dispatch(dfActions.deleteTable(selectedTableForMenu.id)); + } + handleCloseTableMenu(); + }} + disabled={selectedTableForMenu ? tables.some(t => t.derive?.trigger.tableId === selectedTableForMenu.id) : true} + sx={{ fontSize: '12px', display: 'flex', alignItems: 'center', gap: 1, color: 'warning.main' }} + > + + Delete table + + + + {/* Refresh data dialog */} + {selectedTableForRefresh && ( + + )} + + {/* Streaming settings popup */} + {selectedTableForStreamingSettings && ( + manualRefresh(selectedTableForStreamingSettings.id)} + /> + )}     -} + } const VegaLiteChartElement = memo<{ chart: Chart, @@ -1179,7 +1810,22 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { return aOrders.length - bOrders.length; }); - let leafTableGroups = leafTables.reduce((groups: { [groupId: string]: DictTable[] }, leafTable) => { + // Identify hanging tables (tables with no descendants or parents) + let isHangingTable = (table: DictTable) => { + // A table is hanging if: + // 1. It has no derive.source (no parent) + // 2. No other table derives from it (no descendants) + const hasNoParent = table.derive == undefined; + const hasNoDescendants = !tables.some(t => t.derive?.trigger.tableId == table.id); + return hasNoParent && hasNoDescendants; + }; + + // Separate hanging tables from regular leaf tables + let hangingTables = leafTables.filter(t => isHangingTable(t)); + let regularLeafTables = leafTables.filter(t => !isHangingTable(t)); + + // Build groups for regular leaf tables (excluding hanging tables) + let leafTableGroups = regularLeafTables.reduce((groups: { [groupId: string]: DictTable[] }, leafTable) => { // Get the immediate parent table ID (first trigger in the chain) const triggers = getTriggers(leafTable, tables); const immediateParentTableId = triggers.length > 0 ? triggers[triggers.length - 1].tableId : 'root'; @@ -1203,13 +1849,43 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { return groups; }, {}); - let drawerOpen = threadDrawerOpen && leafTables.length > 1; - let collaposedViewWidth = Math.max(...Object.values(leafTableGroups).map(x => x.length)) > 1 ? 248 : 232 + // Filter threads to only include those with length > 1 + let filteredLeafTableGroups: { [groupId: string]: DictTable[] } = {}; + Object.entries(leafTableGroups).forEach(([groupId, groupTables]) => { + // Calculate thread length: count all tables in the thread chain + const threadLength = groupTables.reduce((maxLength, leafTable) => { + const triggers = getTriggers(leafTable, tables); + // Thread length = number of triggers + 1 (the leaf table itself) + return Math.max(maxLength, triggers.length + 1); + }, 0); + + // Only include threads with length > 1 + if (threadLength > 1) { + filteredLeafTableGroups[groupId] = groupTables; + } else { + // Add single-table threads to hanging tables (they go to thread0) + groupTables.forEach(table => { + if (!hangingTables.includes(table)) { + hangingTables.push(table); + } + }); + } + }); + + // Create thread0 group for hanging tables + let thread0Group: { [groupId: string]: DictTable[] } = {}; + if (hangingTables.length > 0) { + thread0Group['thread0'] = hangingTables; + } + + let drawerOpen = threadDrawerOpen && (Object.keys(filteredLeafTableGroups).length > 0 || hangingTables.length > 0); + let allGroupsForWidth = { ...filteredLeafTableGroups, ...thread0Group }; + let collaposedViewWidth = Math.max(...Object.values(allGroupsForWidth).map(x => x.length)) > 1 ? 248 : 232 let view = = function ({ sx }) { p: 1, transition: 'max-width 0.1s linear', // Smooth width transition }}> - {Object.entries(leafTableGroups).map(([groupId, leafTables], i) => { - - let usedIntermediateTableIds = Object.values(leafTableGroups).slice(0, i).flat() + {/* Render thread0 (hanging tables) first if it exists - using compact view */} + {Object.entries(thread0Group).map(([groupId, leafTables], i) => { + return 1 ? '216px' : '200px', + transition: 'all 0.3s ease', + }} /> + })} + {/* Render regular threads (length > 1) */} + {Object.entries(filteredLeafTableGroups).map(([groupId, leafTables], i) => { + // Calculate used tables from thread0 and previous threads + let usedIntermediateTableIds = Object.values(thread0Group).flat() .map(x => [ ...getTriggers(x, tables).map(y => y.tableId) || []]).flat(); - let usedLeafTableIds = Object.values(leafTableGroups).slice(0, i).flat().map(x => x.id); + let usedLeafTableIds = Object.values(thread0Group).flat().map(x => x.id); + + // Add tables from previous regular threads + const previousThreadGroups = Object.values(filteredLeafTableGroups).slice(0, i); + usedIntermediateTableIds = [...usedIntermediateTableIds, ...previousThreadGroups.flat() + .map(x => [ ...getTriggers(x, tables).map(y => y.tableId) || []]).flat()]; + usedLeafTableIds = [...usedLeafTableIds, ...previousThreadGroups.flat().map(x => x.id)]; return = function ({ sx }) { })} + // Calculate total thread count (thread0 + regular threads) + let totalThreadCount = Object.keys(filteredLeafTableGroups).length + (Object.keys(thread0Group).length > 0 ? 1 : 0); + let threadIndices: number[] = []; + if (Object.keys(thread0Group).length > 0) { + threadIndices.push(-1); // thread0 + } + threadIndices.push(...Array.from({length: Object.keys(filteredLeafTableGroups).length}, (_, i) => i)); + let jumpButtonsDrawerOpen = - {_.chunk(Array.from({length: Object.keys(leafTableGroups).length}, (_, i) => i), 3).map((group, groupIdx) => { - const startNum = group[0] + 1; - const endNum = group[group.length - 1] + 1; - const label = startNum === endNum ? `${startNum}` : `${startNum}-${endNum}`; + {_.chunk(threadIndices, 3).map((group, groupIdx) => { + const getLabel = (idx: number) => idx === -1 ? '0' : String(idx + 1); + const startNum = getLabel(group[0]); + const endNum = getLabel(group[group.length - 1]); + const label = startNum === endNum ? startNum : `${startNum}-${endNum}`; return ( @@ -1265,8 +1977,9 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { const currentIndex = Array.from(document.querySelectorAll('[data-thread-index]')).reduce((closest, element) => { const rect = element.getBoundingClientRect(); const distance = Math.abs(rect.left + rect.width/2 - viewportCenter); + const idx = parseInt(element.getAttribute('data-thread-index') || '0'); if (!closest || distance < closest.distance) { - return { index: parseInt(element.getAttribute('data-thread-index') || '0'), distance }; + return { index: idx, distance }; } return closest; }, null as { index: number, distance: number } | null)?.index || 0; @@ -1294,21 +2007,24 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { let jumpButtonDrawerClosed = - {Object.keys(leafTableGroups).map((groupId, idx) => ( - - { - const threadElement = document.querySelector(`[data-thread-index="${idx}"]`); - threadElement?.scrollIntoView({ behavior: 'smooth' }); - }} - > - {idx + 1} - - - ))} + {threadIndices.map((threadIdx) => { + const label = threadIdx === -1 ? '0' : String(threadIdx + 1); + return ( + + { + const threadElement = document.querySelector(`[data-thread-index="${threadIdx}"]`); + threadElement?.scrollIntoView({ behavior: 'smooth' }); + }} + > + {label} + + + ); + })} let jumpButtons = drawerOpen ? jumpButtonsDrawerOpen : jumpButtonDrawerClosed; @@ -1338,7 +2054,7 @@ export const DataThread: FC<{sx?: SxProps}> = function ({ sx }) { { + disabled={totalThreadCount <= 1} onClick={() => { setThreadDrawerOpen(true); }}> diff --git a/src/views/DataView.tsx b/src/views/DataView.tsx index 3d42e455..f1692c5a 100644 --- a/src/views/DataView.tsx +++ b/src/views/DataView.tsx @@ -74,7 +74,7 @@ export const FreeDataViewFC: FC = function DataView() { ? nameSegments.reduce((max, segment) => Math.max(max, segment.length), 0) : name.length; const contentLength = Math.max(maxNameSegmentLength, avgLength); - const minWidth = Math.max(60, contentLength * 8 > 200 ? 200 : contentLength * 8) + 50; // 8px per character with 50px padding + const minWidth = Math.max(60, contentLength * 8 > 240 ? 240 : contentLength * 8) + 50; // 8px per character with 50px padding const width = minWidth; return { minWidth, width }; diff --git a/src/views/DerivedDataDialog.tsx b/src/views/DerivedDataDialog.tsx deleted file mode 100644 index aeb77854..00000000 --- a/src/views/DerivedDataDialog.tsx +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -import { FC } from 'react' -import { - Card, - Box, - Typography, - Dialog, - DialogTitle, - DialogContent, - DialogActions, - Button, - Radio, - styled, - FormControlLabel, - CardContent, - ButtonGroup, -} from '@mui/material'; - -import React from 'react'; - -import { assembleVegaChart } from '../app/utils'; -import { Chart } from '../components/ComponentType'; -import { useSelector } from 'react-redux'; -import { DataFormulatorState } from '../app/dfSlice'; - -import { createDictTable, DictTable } from '../components/ComponentType'; -import { CodeBox } from './VisualizationView'; -import embed from 'vega-embed'; -import { CustomReactTable } from './ReactTable'; - -import DeleteIcon from '@mui/icons-material/Delete'; -import SaveIcon from '@mui/icons-material/Save'; - -export interface DerivedDataDialogProps { - chart: Chart, - candidateTables: DictTable[], - open: boolean, - handleCloseDialog: () => void, - handleSelection: (selectIndex: number) => void, - handleDeleteChart: () => void, - bodyOnly?: boolean, -} - -export const DerivedDataDialog: FC = function DerivedDataDialog({ - chart, candidateTables, open, handleCloseDialog, handleSelection, handleDeleteChart, bodyOnly }) { - - let direction = candidateTables.length > 1 ? "horizontal" : "horizontal" ; - - let [selectionIdx, setSelectionIdx] = React.useState(0); - const conceptShelfItems = useSelector((state: DataFormulatorState) => state.conceptShelfItems); - - let body = - - - {candidateTables.map((table, idx) => { - let code = table.derive?.code || ""; - let extTable = structuredClone(table.rows); - - let assembledChart: any = assembleVegaChart(chart.chartType, chart.encodingMap, conceptShelfItems, extTable, table.metadata); - assembledChart["background"] = "transparent"; - // chart["autosize"] = { - // "type": "fit", - // "contains": "padding" - // }; - const id = `chart-dialog-element-${idx}`; - - const element = - setSelectionIdx(idx)}> - ; - - embed('#' + id, assembledChart, { actions: false, renderer: "canvas" }).then(function (result) { - // Access the Vega view instance (https://vega.github.io/vega/docs/api/view/) as result.view - if (result.view.container()?.getElementsByTagName("canvas")) { - let comp = result.view.container()?.getElementsByTagName("canvas")[0]; - - // Doesn't seem like width & height are actual numbers here on Edge bug - // let width = parseInt(comp?.style.width as string); - // let height = parseInt(comp?.style.height as string); - if (comp) { - const { width, height } = comp.getBoundingClientRect(); - //console.log(`THUMB: width = ${width} height = ${height}`); - if (width > 240 || height > 180) { - let ratio = width / height; - let fixedWidth = width; - if (ratio * 180 < width) { - fixedWidth = ratio * 180; - } - if (fixedWidth > 240) { - fixedWidth = 240; - } - comp?.setAttribute("style", `max-width: 240px; max-height: 180px; width: ${Math.round(fixedWidth)}px; height: ${Math.round(fixedWidth / ratio)}px; `); - } - - } else { - console.log("THUMB: Could not get Canvas HTML5 element") - } - } - }).catch((reason) => { - // console.log(reason) - // console.error(reason) - }); - - let simpleTableView = (t: DictTable) => { - let colDefs = t.names.map(name => { - return { - id: name, label: name, minWidth: 30, align: undefined, - format: (value: any) => `${value}`, source: conceptShelfItems.find(f => f.name == name)?.source - } - }) - return - - - } - - return {setSelectionIdx(idx)}} - sx={{minWidth: "280px", maxWidth: "1920px", display: "flex", flexGrow: 1, margin: "6px", - border: selectionIdx == idx ? "2px solid rgb(2 136 209 / 0.7)": "1px solid rgba(33, 33, 33, 0.1)"}}> - - } - label={{`candidate-${idx+1} (${candidateTables[idx].id})`}} /> - - - {element} - - - - {simpleTableView(createDictTable(table.id, extTable))} - - - - - - - - - })} - - - - if (bodyOnly) { - return - - Transformation from {candidateTables[0].derive?.source} - - {body} - - - - {/* */} - - - - - - ; - } - - return ( - - Derived Data Candidates - - {body} - - - - - - - ); -} \ No newline at end of file diff --git a/src/views/EncodingShelfCard.tsx b/src/views/EncodingShelfCard.tsx index d66e3570..cba27b61 100644 --- a/src/views/EncodingShelfCard.tsx +++ b/src/views/EncodingShelfCard.tsx @@ -63,7 +63,6 @@ import DeleteIcon from '@mui/icons-material/Delete'; import CloseIcon from '@mui/icons-material/Close'; import LightbulbOutlinedIcon from '@mui/icons-material/LightbulbOutlined'; import TipsAndUpdatesIcon from '@mui/icons-material/TipsAndUpdates'; -import BugReportIcon from '@mui/icons-material/BugReport'; import { IdeaChip } from './ChartRecBox'; // Property and state of an encoding shelf @@ -250,111 +249,6 @@ export const TriggerCard: FC<{ } -// Add this component before EncodingShelfCard -const UserActionTableSelector: FC<{ - requiredActionTableIds: string[], - userSelectedActionTableIds: string[], - tables: DictTable[], - updateUserSelectedActionTableIds: (tableIds: string[]) => void, - requiredTableIds?: string[] -}> = ({ requiredActionTableIds, userSelectedActionTableIds, tables, updateUserSelectedActionTableIds, requiredTableIds = [] }) => { - const [anchorEl, setAnchorEl] = useState(null); - const open = Boolean(anchorEl); - - let actionTableIds = [...requiredActionTableIds, ...userSelectedActionTableIds.filter(id => !requiredActionTableIds.includes(id))]; - - const handleClick = (event: React.MouseEvent) => { - setAnchorEl(event.currentTarget); - }; - - const handleClose = () => { - setAnchorEl(null); - }; - - const handleTableSelect = (table: DictTable) => { - if (!actionTableIds.includes(table.id)) { - updateUserSelectedActionTableIds([...userSelectedActionTableIds, table.id]); - } - handleClose(); - }; - - return ( - - {actionTableIds.map((tableId) => { - const isRequired = requiredTableIds.includes(tableId); - return ( - t.id == tableId)?.displayId} - size="small" - sx={{ - height: 16, - fontSize: '10px', - borderRadius: '0px', - bgcolor: isRequired ? 'rgba(25, 118, 210, 0.2)' : 'rgba(25, 118, 210, 0.1)', // darker blue for required - color: 'rgba(0, 0, 0, 0.7)', - '& .MuiChip-label': { - pl: '4px', - pr: '6px' - } - }} - deleteIcon={} - onDelete={isRequired ? undefined : () => updateUserSelectedActionTableIds(actionTableIds.filter(id => id !== tableId))} - /> - ); - })} - - - - - - - - - {tables - .map((table) => { - const isSelected = !!actionTableIds.find(t => t === table.id); - return ( - handleTableSelect(table)} - sx={{ - fontSize: '12px', - display: 'flex', - justifyContent: 'space-between', - alignItems: 'center' - }} - > - {table.displayId} - - ); - }) - } - - - ); -}; - export const EncodingShelfCard: FC = function ({ chartId }) { const theme = useTheme(); @@ -363,7 +257,6 @@ export const EncodingShelfCard: FC = function ({ chartId const tables = useSelector((state: DataFormulatorState) => state.tables); const config = useSelector((state: DataFormulatorState) => state.config); const agentRules = useSelector((state: DataFormulatorState) => state.agentRules); - let existMultiplePossibleBaseTables = tables.filter(t => t.derive == undefined || t.anchored).length > 1; let activeModel = useSelector(dfSelectors.getActiveModel); let allCharts = useSelector(dfSelectors.getAllCharts); @@ -401,9 +294,7 @@ export const EncodingShelfCard: FC = function ({ chartId // Check if chart is available let isChartAvailable = checkChartAvailability(chart, conceptShelfItems, currentTable.rows); - // Add this state - const [userSelectedActionTableIds, setUserSelectedActionTableIds] = useState([]); - + // Consolidated chart state - maps chartId to its ideas, thinkingBuffer, and loading state const [chartState, setChartState] = useState = function ({ chartId })); }; - // Add state for developer message dialog - const [devMessageOpen, setDevMessageOpen] = useState(false); - - // Update the handler to use state - const handleUserSelectedActionTableChange = (newTableIds: string[]) => { - setUserSelectedActionTableIds(newTableIds); - }; - let encodingBoxGroups = Object.entries(ChannelGroups) .filter(([group, channelList]) => channelList.some(ch => Object.keys(encodingMap).includes(ch))) .map(([group, channelList]) => { @@ -479,7 +362,7 @@ export const EncodingShelfCard: FC = function ({ chartId let requiredActionTables = selectBaseTables(activeFields, currentTable, tables); let actionTableIds = [ ...requiredActionTables.map(t => t.id), - ...userSelectedActionTableIds.filter(id => !requiredActionTables.map(t => t.id).includes(id)) + ...tables.filter(t => t.derive === undefined || t.anchored).map(t => t.id).filter(id => !requiredActionTables.map(t => t.id).includes(id)) ]; let getIdeasForVisualization = async () => { @@ -508,24 +391,20 @@ export const EncodingShelfCard: FC = function ({ chartId })); } - // Get the root table (first table in actionTableIds) - const rootTable = tables.find(t => t.id === actionTableIds[0]); - if (!rootTable) { - throw new Error('No root table found'); - } - let chartAvailable = checkChartAvailability(chart, conceptShelfItems, currentTable.rows); let currentChartPng = chartAvailable ? await vegaLiteSpecToPng(assembleVegaChart(chart.chartType, chart.encodingMap, activeFields, currentTable.rows, currentTable.metadata, 20)) : undefined; + let actionTables = actionTableIds.map(id => tables.find(t => t.id == id) as DictTable); + const token = String(Date.now()); const messageBody = JSON.stringify({ token: token, model: activeModel, - input_tables: [{ - name: rootTable.virtual?.tableId || rootTable.id.replace(/\.[^/.]+$/, ""), - rows: rootTable.rows, - attached_metadata: rootTable.attachedMetadata - }], + input_tables: actionTables.map(t => ({ + name: t.virtual?.tableId || t.id.replace(/\.[^/.]+$/, ""), + rows: t.rows, + attached_metadata: t.attachedMetadata + })), language: currentTable.virtual ? "sql" : "python", exploration_thread: explorationThread, current_data_sample: currentTable.rows.slice(0, 10), @@ -1040,7 +919,6 @@ export const EncodingShelfCard: FC = function ({ chartId } -     // Ideas display section - get ideas for current chart @@ -1138,30 +1016,11 @@ export const EncodingShelfCard: FC = function ({ chartId > Editor - - setDevMessageOpen(true)} - sx={{ - width: 20, - height: 20, - fontSize: '10px' - }} - > - - ); let channelComponent = ( - {existMultiplePossibleBaseTables && t.id)} - userSelectedActionTableIds={userSelectedActionTableIds} - tables={tables.filter(t => t.derive === undefined || t.anchored)} - updateUserSelectedActionTableIds={handleUserSelectedActionTableChange} - requiredTableIds={requiredActionTables.map(t => t.id)} - />} + fileInputRef.current?.click()} + > + + + Drag & drop file here + + + or Browse + + + Supported: CSV, TSV, JSON, Excel (xlsx, xls) + + + + )} + + + + setUrlContent(e.target.value.trim())} + disabled={isLoading} + error={urlContent !== '' && !hasValidUrlSuffix} + helperText={urlContent !== '' && !hasValidUrlSuffix ? 'URL should link to a .csv, .tsv, or .json file' : ''} + size="small" + sx={{ + '& .MuiInputBase-input': { + fontSize: '0.875rem', + }, + '& .MuiInputBase-input::placeholder': { + fontSize: '0.875rem', + }, + '& .MuiFormHelperText-root': { + fontSize: '0.75rem', + }, + }} + /> + + + + + {tabValue === 0 && ( + + )} + {tabValue === 2 && ( + + )} + + + ); +}; diff --git a/src/views/ReportView.tsx b/src/views/ReportView.tsx index 22562c27..e1ef7381 100644 --- a/src/views/ReportView.tsx +++ b/src/views/ReportView.tsx @@ -51,6 +51,7 @@ import { Collapse } from '@mui/material'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import ExpandLessIcon from '@mui/icons-material/ExpandLess'; import { convertToChartifact, openChartifactViewer } from './ChartifactDialog'; +import StreamIcon from '@mui/icons-material/Stream'; // Typography constants const FONT_FAMILY_SYSTEM = '-apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, "Apple Color Emoji", Arial, sans-serif, "Segoe UI Emoji", "Segoe UI Symbol"'; @@ -231,13 +232,13 @@ export const ReportView: FC = () => { const [isLoadingPreviews, setIsLoadingPreviews] = useState(false); const [isGenerating, setIsGenerating] = useState(false); const [error, setError] = useState(''); - const [style, setStyle] = useState('short note'); + const [style, setStyle] = useState('social post'); const [mode, setMode] = useState<'compose' | 'post'>(allGeneratedReports.length > 0 ? 'post' : 'compose'); // Local state for current report const [currentReportId, setCurrentReportId] = useState(undefined); const [generatedReport, setGeneratedReport] = useState(''); - const [generatedStyle, setGeneratedStyle] = useState('short note'); + const [generatedStyle, setGeneratedStyle] = useState('social post'); const [cachedReportImages, setCachedReportImages] = useState>({}); const [shareButtonSuccess, setShareButtonSuccess] = useState(false); const [hideTableOfContents, setHideTableOfContents] = useState(false); @@ -368,6 +369,67 @@ export const ReportView: FC = () => { } }, [currentReportId]); + // Auto-refresh chart images when underlying table data changes + // This enables real-time chart updates in reports when data is streaming + const tableRowSignaturesRef = useRef>(new Map()); + + useEffect(() => { + if (!currentReportId || mode !== 'post') return; + + const currentReport = allGeneratedReports.find(r => r.id === currentReportId); + if (!currentReport) return; + + // Get all tables referenced by the report's charts + const reportChartIds = currentReport.selectedChartIds; + const affectedTableIds = new Set(); + + reportChartIds.forEach(chartId => { + const chart = charts.find(c => c.id === chartId); + if (chart) { + affectedTableIds.add(chart.tableRef); + } + }); + + // Check if any affected tables have changed + let hasChanges = false; + affectedTableIds.forEach(tableId => { + const table = tables.find(t => t.id === tableId); + if (table) { + // Use contentHash if available (set by state management), otherwise fallback to lightweight rowCount + // This avoids expensive JSON.stringify operations on every table change during streaming updates + const signature = table.contentHash || `${table.rows.length}`; + + const prevSignature = tableRowSignaturesRef.current.get(tableId); + if (prevSignature && prevSignature !== signature) { + hasChanges = true; + } + tableRowSignaturesRef.current.set(tableId, signature); + } + }); + + // If data changed, regenerate chart images for the report + if (hasChanges) { + + reportChartIds.forEach(chartId => { + const chart = charts.find(c => c.id === chartId); + if (!chart) return; + + const chartTable = tables.find(t => t.id === chart.tableRef); + if (!chartTable) return; + + if (chart.chartType === 'Table' || chart.chartType === '?') { + return; + } + + getChartImageFromVega(chart, chartTable).then(({ blobUrl, width, height }) => { + if (blobUrl) { + updateCachedReportImages(chart.id, blobUrl, width, height); + } + }); + }); + } + }, [tables, currentReportId, mode, allGeneratedReports, charts]); + // Sort charts based on data thread ordering @@ -734,7 +796,7 @@ export const ReportView: FC = () => { // No reports left, clear the view and go back to compose mode setCurrentReportId(undefined); setGeneratedReport(''); - setGeneratedStyle('short note'); + setGeneratedStyle('social post'); setMode('compose'); } } @@ -832,7 +894,7 @@ export const ReportView: FC = () => { }} > {[ - { value: 'short note', label: 'short note' }, + { value: 'live report', label: 'live report' }, { value: 'blog post', label: 'blog post' }, { value: 'social post', label: 'social post' }, { value: 'executive summary', label: 'executive summary' }, @@ -848,7 +910,7 @@ export const ReportView: FC = () => { minWidth: 'auto' }} > - {option.label} + {option.value === 'live report' ? : <>} {option.label} ))} diff --git a/src/views/SelectableDataGrid.tsx b/src/views/SelectableDataGrid.tsx index 5315022c..fcf04731 100644 --- a/src/views/SelectableDataGrid.tsx +++ b/src/views/SelectableDataGrid.tsx @@ -20,12 +20,18 @@ import { getIconFromType } from './ViewUtils'; import { IconButton, TableSortLabel, Typography } from '@mui/material'; import _ from 'lodash'; -import { FieldSource } from '../components/ComponentType'; +import { FieldSource, FieldItem } from '../components/ComponentType'; import FileDownloadIcon from '@mui/icons-material/FileDownload'; import CloudQueueIcon from '@mui/icons-material/CloudQueue'; import CasinoIcon from '@mui/icons-material/Casino'; +import ArrowUpwardIcon from '@mui/icons-material/ArrowUpward'; +import ArrowDownwardIcon from '@mui/icons-material/ArrowDownward'; +import UnfoldMoreIcon from '@mui/icons-material/UnfoldMore'; import { getUrls } from '../app/utils'; +import { useDrag } from 'react-dnd'; +import { useSelector } from 'react-redux'; +import { DataFormulatorState } from '../app/dfSlice'; export interface ColumnDef { id: string; @@ -69,6 +75,170 @@ function getComparator( : (a, b) => -descendingComparator(a, b, orderBy); } +// Get color for field source hover highlight +function getColorForFieldSource(source: string | undefined, theme: any): string { + if (!source) { + return theme.palette.primary.main; // Default to primary for original fields + } + + switch (source) { + case "derived": + return theme.palette.derived.main; // Yellow/gold for derived fields + case "custom": + return theme.palette.custom.main; // Orange for custom fields + case "original": + default: + return theme.palette.primary.main; // Blue for original fields + } +} + +// Draggable header component +interface DraggableHeaderProps { + columnDef: ColumnDef; + orderBy: string | undefined; + order: 'asc' | 'desc'; + onSortClick: () => void; + tableId: string; +} + +const DraggableHeader: React.FC = ({ + columnDef, orderBy, order, onSortClick, tableId +}) => { + const theme = useTheme(); + const conceptShelfItems = useSelector((state: DataFormulatorState) => state.conceptShelfItems); + + // Find the corresponding FieldItem for this column + // Try to find by name first, then by constructing the ID for original fields + const field = conceptShelfItems.find(f => f.name === columnDef.id) || + conceptShelfItems.find(f => f.id === `original--${tableId}--${columnDef.id}`); + + // Only make draggable if we have a field + // react-dnd has a drag threshold, so clicks will still work for sorting + const [{ isDragging }, dragSource, dragPreview] = useDrag(() => ({ + type: "concept-card", + item: field ? { + type: 'concept-card', + fieldID: field.id, + source: "conceptShelf" + } : undefined, + canDrag: !!field, + collect: (monitor) => ({ + isDragging: monitor.isDragging(), + handlerId: monitor.getHandlerId(), + }), + }), [field]); + + let backgroundColor = "white"; + let borderBottomColor = theme.palette.primary.main; + if (columnDef.source == "custom") { + backgroundColor = alpha(theme.palette.custom.main, 0.05); + borderBottomColor = theme.palette.custom.main; + } else { + backgroundColor = alpha(theme.palette.primary.main, 0.05); + borderBottomColor = theme.palette.primary.main; + } + + const opacity = isDragging ? 0.3 : 1; + const cursorStyle = field ? (isDragging ? "grabbing" : "grab") : "default"; + + // Enhanced background color on hover for draggable headers - based on field source (derived/original) + const hoverBackgroundColor = field + ? alpha(getColorForFieldSource(field.source, theme), 0.1) + : backgroundColor; + + // Determine sort icon + const getSortIcon = () => { + if (orderBy !== columnDef.id) { + return ; + } + return order === 'asc' + ? + : ; + }; + + return ( + + {/* Main content area - draggable for concepts, using original TableSortLabel structure */} + { + // Prevent sort when dragging + if (!isDragging) { + e.stopPropagation(); + onSortClick(); + } + }} + > + + {getIconFromType(columnDef.dataType)} + + + {columnDef.label} + + + {/* Separate sort handler button */} + Sort by {columnDef.label}}> + { + e.stopPropagation(); + onSortClick(); + }} + sx={{ + padding: '2px', + marginLeft: '4px', + marginRight: '2px', + opacity: orderBy === columnDef.id ? 1 : 0.5, + '&:hover': { + opacity: 1, + backgroundColor: alpha(theme.palette.action.hover, 0.2), + }, + }} + > + {getSortIcon()} + + + + ); +}; + export const SelectableDataGrid: React.FC = ({ tableId, rows, tableName, columnDefs, rowCount, virtual }) => { @@ -192,16 +362,6 @@ export const SelectableDataGrid: React.FC = ({ return ( {columnDefs.map((columnDef, index) => { - let backgroundColor = "white"; - let borderBottomColor = theme.palette.primary.main; - if (columnDef.source == "custom") { - backgroundColor = alpha(theme.palette.custom.main, 0.05); - borderBottomColor = theme.palette.custom.main; - } else { - backgroundColor = "white"; - borderBottomColor = theme.palette.primary.main; - } - return ( = ({ align={columnDef.align} sx={{p: 0, minWidth: columnDef.minWidth, width: columnDef.width,}} > - - - { - let newOrder: 'asc' | 'desc' = 'asc'; - let newOrderBy : string | undefined = columnDef.id; - if (orderBy === columnDef.id && order === 'asc') { - newOrder = 'desc'; - } else if (orderBy === columnDef.id && order === 'desc') { - newOrder = 'asc'; - newOrderBy = undefined; - } else { - newOrder = 'asc'; - } + { + let newOrder: 'asc' | 'desc' = 'asc'; + let newOrderBy : string | undefined = columnDef.id; + if (orderBy === columnDef.id && order === 'asc') { + newOrder = 'desc'; + } else if (orderBy === columnDef.id && order === 'desc') { + newOrder = 'asc'; + newOrderBy = undefined; + } else { + newOrder = 'asc'; + } - setOrder(newOrder); - setOrderBy(newOrderBy); - - if (virtual) { - fetchVirtualData(newOrderBy ? [newOrderBy] : [], newOrder); - } - }} - > - - {getIconFromType(columnDef.dataType)} - - - {columnDef.label} - - - - + setOrder(newOrder); + setOrderBy(newOrderBy); + + if (virtual) { + fetchVirtualData(newOrderBy ? [newOrderBy] : [], newOrder); + } + }} + /> ); })} @@ -256,12 +404,12 @@ export const SelectableDataGrid: React.FC = ({ return ( <> {columnDefs.map((column, colIndex) => { - let backgroundColor = "white"; - if (column.source == "custom") { - backgroundColor = alpha(theme.palette.custom.main, 0.05); - } else { - backgroundColor = "rgba(255,255,255,0.05)"; - } + let backgroundColor = "rgba(255,255,255,0.05)"; + // if (column.source == "custom") { + // backgroundColor = alpha(theme.palette.custom.main, 0.03); + // } else { + // backgroundColor = "rgba(255,255,255,0.05)"; + // } return (