From d81e3713ac9f783435cdf93e0242e82aa835787a Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Mon, 22 Jun 2020 12:01:57 +0100 Subject: [PATCH 01/10] Add top-level filter to the tasks table. Remove the inline table header filters as state is lost on refresh. --- frontend/src/TasksDashboard.jsx | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/frontend/src/TasksDashboard.jsx b/frontend/src/TasksDashboard.jsx index 712ae0d..a8f7ed7 100644 --- a/frontend/src/TasksDashboard.jsx +++ b/frontend/src/TasksDashboard.jsx @@ -5,6 +5,7 @@ import Typography from '@material-ui/core/Typography'; import CircularProgress from '@material-ui/core/CircularProgress'; import Button from '@material-ui/core/Button'; import FormControlLabel from '@material-ui/core/FormControlLabel'; +import TextField from '@material-ui/core/TextField'; import Checkbox from '@material-ui/core/Checkbox'; import pretty from 'pretty-time'; @@ -32,6 +33,22 @@ const mapDispatchToProps = { loadNodes, }; +const filterTaskTableData = (taskDatas, tasksFilter) => { + if (!tasksFilter) { + return taskDatas; + } + + return taskDatas.filter(taskData => { + if (taskData.action.indexOf(tasksFilter) > 0) { + return true; + } + if (taskData.type.indexOf(tasksFilter) > 0) { + return true; + } + return false; + }); +}; + const generateTaskTableData = (taskDatas, includeChildren) => { return taskDatas.reduce((rows, taskData) => { let childRows = []; @@ -84,6 +101,7 @@ class TasksDashboard extends React.Component { state = { showChildren: false, + tasksFilter: '', }; componentDidMount() { @@ -150,28 +168,24 @@ class TasksDashboard extends React.Component { dataKey: 'id', width: 300, sortable: true, - searchable: true, }, { title: 'Type', dataKey: 'type', width: 300, sortable: true, - searchable: true, }, { title: 'Action', dataKey: 'action', width: 500, sortable: true, - searchable: true, }, { title: 'Node', dataKey: 'node', width: 300, sortable: true, - searchable: true, formatter: nodeId => { const nodeData = nodes.data.nodes[nodeId]; return nodeData ? nodeData.name : nodeId; @@ -193,7 +207,7 @@ class TasksDashboard extends React.Component { ]; const tableData = generateTaskTableData( - Object.values(tasks.data.tasks), + filterTaskTableData(Object.values(tasks.data.tasks), this.state.tasksFilter), this.state.showChildren, ); @@ -217,6 +231,15 @@ class TasksDashboard extends React.Component { } label="Show child tasks" /> + this.setState({tasksFilter: ev.target.value})} + /> + } + label="Filter tasks" + />
From 4ab33a0a4af9b9c9221ea7cb093d39b59d15f874 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Mon, 22 Jun 2020 12:02:12 +0100 Subject: [PATCH 02/10] Add auto refresh dropdown to the tasks dashboard. --- frontend/src/TasksDashboard.jsx | 35 ++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/frontend/src/TasksDashboard.jsx b/frontend/src/TasksDashboard.jsx index a8f7ed7..136cf63 100644 --- a/frontend/src/TasksDashboard.jsx +++ b/frontend/src/TasksDashboard.jsx @@ -7,6 +7,8 @@ import Button from '@material-ui/core/Button'; import FormControlLabel from '@material-ui/core/FormControlLabel'; import TextField from '@material-ui/core/TextField'; import Checkbox from '@material-ui/core/Checkbox'; +import Select from '@material-ui/core/Select'; +import MenuItem from '@material-ui/core/MenuItem'; import pretty from 'pretty-time'; import { loadTasks } from './data/tasks/actions'; @@ -102,6 +104,7 @@ class TasksDashboard extends React.Component { state = { showChildren: false, tasksFilter: '', + refreshInterval: 0, }; componentDidMount() { @@ -119,11 +122,24 @@ class TasksDashboard extends React.Component { } } - componentDidUpdate(prevProps) { + componentWillUnmount() { + if (this.refreshIntervalTimer) { + clearInterval(this.refreshIntervalTimer); + } + } + + componentDidUpdate(prevProps, prevState) { if (prevProps.clusters.currentCluster !== this.props.clusters.currentCluster) { this.props.loadNodes(); this.props.loadTasks(); } + + if (prevState.refreshInterval !== this.state.refreshInterval) { + this.refreshIntervalTimer = setInterval( + this.props.loadTasks, + this.state.refreshInterval * 1000, + ); + } } getContainerStyles(dataLoaded) { @@ -214,6 +230,7 @@ class TasksDashboard extends React.Component { return
this.setState({refreshInterval: ev.target.value})} + > + Disabled + 10s + 30s + 60s + + } + /> + Date: Mon, 22 Jun 2020 12:02:22 +0100 Subject: [PATCH 03/10] Don't send up node_modules bloat into docker image. --- .dockerignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..85fddf4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +frontend/node_modules/ +frontend/build/ From b77245539e49262d3a3c8f28d3ea1f67eb63cc20 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Mon, 22 Jun 2020 12:45:05 +0100 Subject: [PATCH 04/10] Clear any interval before re-creating on refresh change. --- frontend/src/TasksDashboard.jsx | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/frontend/src/TasksDashboard.jsx b/frontend/src/TasksDashboard.jsx index 136cf63..91bf142 100644 --- a/frontend/src/TasksDashboard.jsx +++ b/frontend/src/TasksDashboard.jsx @@ -123,9 +123,7 @@ class TasksDashboard extends React.Component { } componentWillUnmount() { - if (this.refreshIntervalTimer) { - clearInterval(this.refreshIntervalTimer); - } + this.clearAnyRefreshInterval(); } componentDidUpdate(prevProps, prevState) { @@ -135,10 +133,19 @@ class TasksDashboard extends React.Component { } if (prevState.refreshInterval !== this.state.refreshInterval) { - this.refreshIntervalTimer = setInterval( - this.props.loadTasks, - this.state.refreshInterval * 1000, - ); + this.clearAnyRefreshInterval(); + if (this.state.refreshInterval) { + this.refreshIntervalTimer = setInterval( + this.props.loadTasks, + this.state.refreshInterval * 1000, + ); + } + } + } + + clearAnyRefreshInterval() { + if (this.refreshIntervalTimer) { + clearInterval(this.refreshIntervalTimer); } } From aec7c28cfffc6318ec0571b59fb4fc82928a26fe Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Mon, 22 Jun 2020 12:45:20 +0100 Subject: [PATCH 05/10] Always show filters, nest data errors/loading inside table area. --- frontend/src/TasksDashboard.jsx | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/frontend/src/TasksDashboard.jsx b/frontend/src/TasksDashboard.jsx index 91bf142..68fe762 100644 --- a/frontend/src/TasksDashboard.jsx +++ b/frontend/src/TasksDashboard.jsx @@ -169,12 +169,11 @@ class TasksDashboard extends React.Component { }); }; - render() { + renderTable() { const { tasks, nodes } = this.props; + if (isNotLoaded(tasks) || isLoading(tasks) || isNotLoaded(nodes) || isLoading(nodes)) { - return
- -
; + return ; } if (isErrored(tasks) || isErrored(nodes)) { @@ -234,6 +233,10 @@ class TasksDashboard extends React.Component { this.state.showChildren, ); + return ; + } + + render() { return
this.setState({refreshInterval: ev.target.value})} > Disabled + 5s 10s 30s 60s @@ -266,6 +270,7 @@ class TasksDashboard extends React.Component { control={ } @@ -283,7 +288,7 @@ class TasksDashboard extends React.Component {
-
+ {this.renderTable()} ; } From 413f36a631fcd20240634a348ffb5424a2698916 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Mon, 22 Jun 2020 13:10:58 +0100 Subject: [PATCH 06/10] Include detailed task information, and format specific tasks. Currently just formats reindex tasks into a status string. --- .../elasticsurgery/views/clusters_proxy.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/backend/elasticsurgery/views/clusters_proxy.py b/backend/elasticsurgery/views/clusters_proxy.py index 41513f7..4f5205e 100644 --- a/backend/elasticsurgery/views/clusters_proxy.py +++ b/backend/elasticsurgery/views/clusters_proxy.py @@ -42,5 +42,24 @@ def put_cluster_settings(cluster_slug): @app.route('/api/clusters//tasks', methods=('GET',)) @pass_cluster_client def get_cluster_tasks(cluster_client): - tasks = cluster_client.tasks.list(group_by='parents') - return jsonify(**tasks) + tasks = cluster_client.tasks.list(group_by='parents', detailed=True) + + def process_task(task): + if task['action'] == 'indices:data/write/reindex': + status = task['status'] + percentage_complete = round(( + (status['created'] + status['updated']) / status['total'] + * 100 + ), 2) + task['status_string'] = ( + f'(created={status["created"]} + updated={status["updated"]})' + f'/{status["total"]} ({percentage_complete}%)' + ) + + return task + + tasks = { + task_id: process_task(task) + for task_id, task in tasks['tasks'].items() + } + return jsonify(tasks=tasks) From 430c08c834211bb1055b7e5253aaefc9316c6d67 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Mon, 22 Jun 2020 13:11:07 +0100 Subject: [PATCH 07/10] Show description + status string in task table. --- frontend/src/TasksDashboard.jsx | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/frontend/src/TasksDashboard.jsx b/frontend/src/TasksDashboard.jsx index 68fe762..7f731fc 100644 --- a/frontend/src/TasksDashboard.jsx +++ b/frontend/src/TasksDashboard.jsx @@ -188,18 +188,30 @@ class TasksDashboard extends React.Component { { title: 'ID', dataKey: 'id', - width: 300, + width: 100, sortable: true, }, { title: 'Type', dataKey: 'type', - width: 300, + width: 100, sortable: true, }, { title: 'Action', dataKey: 'action', + width: 300, + sortable: true, + }, + { + title: 'Description', + dataKey: 'description', + width: 500, + sortable: true, + }, + { + title: 'Status', + dataKey: 'status_string', width: 500, sortable: true, }, From 2a99b5167129feb5a914932785879e705b07dc22 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Tue, 23 Jun 2020 11:22:23 +0100 Subject: [PATCH 08/10] Make it possible to set settings via environment variables. --- backend/elasticsurgery/settings.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/backend/elasticsurgery/settings.py b/backend/elasticsurgery/settings.py index e35ae36..d342cf3 100644 --- a/backend/elasticsurgery/settings.py +++ b/backend/elasticsurgery/settings.py @@ -1,8 +1,11 @@ -# TODO: Open-source shouldn't store settings like this. -SECRET_KEY = '5xP9_fMwqRnF047ZEwnQd9nCLknz3TrdQvggugBMxlo0zT3yEJ0f3g' -DEBUG = True +from os import environ -ES_STATE_HOSTS = ['elasticsurgery-elasticsearch:9200'] -ES_CLUSTERS_INDEX_NAME = '.elasticsurgery-clusters' -ES_LOGS_INDEX_NAME = '.elasticsurgery-logs' +SECRET_KEY = environ.get('SECRET_KEY', 'not-a-secret') + +DEBUG = environ.get('DEBUG') == 'on' or environ.get('KTD_ENV') + +ES_STATE_HOSTS = environ.get('ES_STATE_HOSTS', 'elasticsurgery-elasticsearch:9200').split(',') + +ES_CLUSTERS_INDEX_NAME = environ.get('ES_CLUSTERS_INDEX_NAME', '.elasticsurgery-clusters') +ES_LOGS_INDEX_NAME = environ.get('ES_LOGS_INDEX_NAME', '.elasticsurgery-logs') From d74996d8aef12279d2b751460dbad6ee50e623af Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Tue, 23 Jun 2020 11:29:38 +0100 Subject: [PATCH 09/10] Fix uwsgi config (needs chdir=backend). --- docker/uwsgi.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/uwsgi.conf b/docker/uwsgi.conf index daa9bc1..3098e09 100644 --- a/docker/uwsgi.conf +++ b/docker/uwsgi.conf @@ -10,6 +10,7 @@ http-socket = 0.0.0.0:80 # App plugin = python module = boot:app +chdir = backend # Worker processes -processes = 4 \ No newline at end of file +processes = 4 From 3f1c00c0c33e4cd8f07bacfd91188a081c9ccbe3 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Tue, 23 Jun 2020 11:30:29 +0100 Subject: [PATCH 10/10] Update readme. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9351d53..635c93d 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ElasticSurgery is an Elastcsearch cluster _management interface_. Designed to vi ## Development -Setup your environment like so - you'll need [`kubetools-client`](https://github.com/EDITD/kubetools-client) installed: +Setup your environment like so - you'll need [`kubetools`](https://github.com/EDITD/kubetools) installed: ```sh # Install the node modules