diff --git a/dashboard/.gitignore b/dashboard/.gitignore new file mode 100644 index 00000000..52f6d3d0 --- /dev/null +++ b/dashboard/.gitignore @@ -0,0 +1 @@ +dashboard3.min.* diff --git a/dashboard/assets/dashboard3/Dashboard.hx b/dashboard/assets/dashboard3/Dashboard.hx index 7558a0b6..1a31b7cd 100644 --- a/dashboard/assets/dashboard3/Dashboard.hx +++ b/dashboard/assets/dashboard3/Dashboard.hx @@ -3,6 +3,7 @@ using StringTools; import Std; import haxe.ds.StringMap; import js.Browser; +import js.Syntax.code; import js.html.XMLHttpRequest; import haxe.Json; import Reflect; @@ -28,6 +29,9 @@ class LogLine { } class Job { + public var clusterize:Dynamic; + private var clusterizeRows:Array = []; + public var ident : String; public var logLines : Array = []; public var aborted : Bool; @@ -90,6 +94,46 @@ class Job { return sum / 60.0; } + private function getTextColor(logLine):String { + // response codes: + // 200 OK + if (logLine.responseCode == 200) { + return "text-success"; + } + // 100s + if (logLine.responseCode >= 100 && logLine.responseCode < 200) { + return "text-primary"; + } + // 200s (we already checked 200) + if (logLine.responseCode >= 201 && logLine.responseCode < 300) { + return "text-success-emphasis"; + } + // 300s + if (logLine.responseCode >= 300 && logLine.responseCode < 400) { + return "text-info"; + } + // 400s + if (logLine.responseCode >= 400 && logLine.responseCode < 500) { + return "text-warning"; + } + // 500s + if (logLine.responseCode >= 500 && logLine.responseCode < 600) { + return "text-danger"; + } + + // warning levels / misc. + if (logLine.isWarning) { + return "text-warning"; + } + if (logLine.isError) { + return "text-danger"; + } + if (logLine.message != null || logLine.pattern != null) { + return "text-muted"; + } + return ""; // fallback, no coloring + } + public function consumeLogEvent(logEvent:Dynamic, maxScrollback:Int) { var jobData:Dynamic = logEvent.job_data; @@ -135,8 +179,8 @@ class Job { totalResponses = r1xx + r2xx + r3xx + r4xx + r5xx + errorCount; queueRemaining = itemsQueued - itemsDownloaded; - if (logLines.length >= maxScrollback) { - logLines.shift(); + if (logLines.length > maxScrollback) { + logLines = logLines.slice(logLines.length - maxScrollback); } fillDownloadCountBucket(); @@ -146,78 +190,92 @@ class Job { pendingLogLines += 1; } - public function drawPendingLogLines() { + public function drawPendingLogLines(maxScrollback:Int) { if (pendingLogLines <= 0) { return; } - var logElement = Browser.document.getElementById('job-log-${ident}'); + var scrollEl = Browser.document.getElementById('job-log-${ident}'); + var contentEl = Browser.document.getElementById('job-log-content-${ident}'); - if (logElement == null) { + if (scrollEl == null || contentEl == null) { + if (clusterize != null) { + untyped clusterize.destroy(true); + clusterize = null; + } return; } - for (logLine in logLines.slice(-pendingLogLines)) { - var logLineDiv = Browser.document.createDivElement(); - - logLineDiv.className = "job-log-line"; - - if (logLine.responseCode == 200) { - logLineDiv.classList.add("text-success"); - } else if (logLine.isWarning) { - logLineDiv.classList.add("bg-warning"); - } else if (logLine.isError) { - logLineDiv.classList.add("bg-danger"); - } else if (logLine.message != null || logLine.pattern != null) { - logLineDiv.classList.add("text-muted"); + if (clusterize != null) { + var stale = + (untyped clusterize.scroll_elem != scrollEl) || + (untyped clusterize.content_elem != contentEl); + if (stale) { + untyped clusterize.destroy(true); + clusterize = null; } + } + + if (clusterize == null) { + clusterizeRows = []; + var clusterizeOptions = { + rows: [], + scrollId: 'job-log-${ident}', + contentId: 'job-log-content-${ident}' + }; + clusterize = code("new Clusterize")(clusterizeOptions); + } + + for (logLine in logLines.slice(-pendingLogLines)) { + var logColor = getTextColor(logLine); + var text = ""; if (logLine.responseCode > 0 || logLine.wgetCode != null) { - var text; if (logLine.responseCode > 0) { - text = '${logLine.responseCode} '; + text += '${logLine.responseCode} '; } else { - text = '${logLine.wgetCode} '; + text += '${logLine.wgetCode} '; } - logLineDiv.appendChild(Browser.document.createTextNode(text)); } - if (logLine.url != null) { - var element = Browser.document.createAnchorElement(); - element.href = logLine.url; - element.textContent = logLine.url; - element.className = "job-log-line-url"; - logLineDiv.appendChild(element); + if (logLine.url != null) { + text += '' + logLine.url + ''; if (logLine.pattern != null) { - var element = Browser.document.createSpanElement(); - element.textContent = logLine.pattern; - element.className = "text-warning"; - logLineDiv.appendChild(Browser.document.createTextNode(" ")); - logLineDiv.appendChild(element); + text += ' ' + logLine.pattern + ''; } } else if (logLine.message != null) { - logLineDiv.textContent = logLine.message; - logLineDiv.classList.add("job-log-line-message"); + text += '' + logLine.message + ''; } - logElement.appendChild(logLineDiv); - } + if (logColor != "") { + text = '
' + text + '
'; + } else { + text = '
' + text + '
'; + } - var numToTrim = logElement.childElementCount - logLines.length; + clusterizeRows.push(text); + } - if (numToTrim > 0) { - for (dummy in 0...numToTrim) { - var child = logElement.firstChild; - if (child != null) { - logElement.removeChild(child); - } - } + if (clusterizeRows.length > maxScrollback) { + clusterizeRows = clusterizeRows.slice(clusterizeRows.length - maxScrollback); } - logElement.setAttribute("data-autoscroll-dirty", "true"); + untyped clusterize.update(clusterizeRows); pendingLogLines = 0; } + public function enforceScrollback(maxScrollback:Int) { + if (logLines.length > maxScrollback) { + logLines = logLines.slice(logLines.length - maxScrollback); + } + if (clusterizeRows.length > maxScrollback) { + clusterizeRows = clusterizeRows.slice(clusterizeRows.length - maxScrollback); + if (clusterize != null) { + untyped clusterize.update(clusterizeRows); + } + } + } + public function attachAntiScroll() { var logWindow = Browser.document.getElementById('job-log-${ident}'); @@ -261,7 +319,7 @@ class Job { class Dashboard { - var angular = untyped __js__("angular"); + var angular = code("angular"); var app:Dynamic; var jobs:Array = []; var jobMap:StringMap = new StringMap(); @@ -318,6 +376,9 @@ class Dashboard { scope.sortParam = "startedAt"; scope.showNicks = showNicks; scope.drawInterval = drawInterval; + scope.currentPage = 1; + scope.pageSize = 20; + scope.totalPages = 1; dashboardControllerScopeApply = Reflect.field(scope, "$apply").bind(scope); scope.filterOperator = function (job:Job) { var query:String = scope.filterQuery; @@ -334,6 +395,36 @@ class Dashboard { scope.applyFilterQuery = function (query:String) { scope.filterQuery = query; } + scope.$watchGroup(["filterQuery", "jobs.length"], function(newVals:Dynamic, oldVals:Dynamic) { + var filterQuery = newVals[0]; + + // update max scrollback when searching + var maxScrollback = 500; + if (filterQuery == null || filterQuery.trim() == "") { + maxScrollback = 50; + } + changeMaxScrollback(maxScrollback); + for (job in jobs) { + job.enforceScrollback(this.maxScrollback); + } + + // pagination + var filtered = scope.jobs.filter(scope.filterOperator); + var pages:Float = Math.ceil(filtered.length / (scope.pageSize : Float)); + scope.totalPages = Std.int(Math.max(1, pages)); + if (oldVals != null && filterQuery != oldVals[0]) { + scope.currentPage = 1; + } else { + if (scope.currentPage > scope.totalPages) { + scope.currentPage = scope.totalPages; + } + } + }); + scope.setPage = function (page:Int) { + if (page >= 1 && page <= scope.totalPages) { + scope.currentPage = page; + } + }; } ]; @@ -355,10 +446,15 @@ class Dashboard { return args; } + public function changeMaxScrollback(maxScrollback:Int) { + this.maxScrollback = maxScrollback; + return; + } + public static function main() { var args = getQueryArgs(); var hostname; - var maxScrollback = 20; + var maxScrollback = 50; var showNicks = args.exists("showNicks"); if (args.exists("host")) { @@ -367,10 +463,6 @@ class Dashboard { hostname = Browser.location.hostname; } - if (Browser.navigator.userAgent.indexOf("Mobi") == -1) { - maxScrollback = 500; - } - var dashboard = new Dashboard(hostname, maxScrollback, showNicks); dashboard.run(); } @@ -434,16 +526,16 @@ class Dashboard { websocket = null; showError("Lost connection. Reconnecting..."); - untyped __js__("setTimeout")(function () { + code("setTimeout")(function () { openWebSocket(); }, 60000); } websocket.onerror = websocket.onclose; } - private function scheduleDraw(delayMS:Int = 1000) { - drawTimerHandle = untyped __js__("setTimeout")(function () { - var delay:Int = dashboardControllerScope.drawInterval; + private function scheduleDraw(delayMS:Float = 1000) { + drawTimerHandle = code("setTimeout")(function () { + var delay:Float = dashboardControllerScope.drawInterval; if (!Browser.document.hidden && !dashboardControllerScope.paused) { var beforeDate = Date.now(); @@ -495,7 +587,7 @@ class Dashboard { for (job in jobs) { if (!job.logPaused) { - job.drawPendingLogLines(); + job.drawPendingLogLines(maxScrollback); } } @@ -503,17 +595,13 @@ class Dashboard { } private function scrollLogsToBottom() { - var nodes = Browser.document.querySelectorAll("[data-autoscroll-dirty].autoscroll"); - var pending = new Array(); - - for (node in nodes) { - var element:Element = cast(node, Element); - element.removeAttribute("data-autoscroll-dirty"); - pending.push(element); - } - for (element in pending) { - // Try to do layout in a tight loop - element.scrollTop = 99999; + for (job in jobs) { + if (!job.logPaused && job.clusterize != null) { + var scrollEl = Browser.document.getElementById('job-log-${job.ident}'); + if (scrollEl != null) { + scrollEl.scrollTop = scrollEl.scrollHeight; + } + } } } } diff --git a/dashboard/assets/dashboard3/Makefile b/dashboard/assets/dashboard3/Makefile index 164dd59e..11a32051 100644 --- a/dashboard/assets/dashboard3/Makefile +++ b/dashboard/assets/dashboard3/Makefile @@ -1,4 +1,10 @@ -all: dashboard +all: dashboard minify dashboard: Dashboard.hx - haxe -main Dashboard -js dashboard3.js + haxe -main Dashboard -D analyzer-optimize -dce full -D js-es=6 -js dashboard3.js + +minify: dashboard3.js dashboard3.css ../../dashboard3.html +# npm i -g terser csso-cli html-minifier-next + terser dashboard3.js -c -m --ecma 2015 -o dashboard3.min.js + csso dashboard3.css -o dashboard3.min.css + html-minifier-next --collapse-whitespace --minify-js true ../../dashboard3.html -o ../../dashboard3.min.html diff --git a/dashboard/assets/dashboard3/dashboard3.css b/dashboard/assets/dashboard3/dashboard3.css new file mode 100644 index 00000000..61b981c5 --- /dev/null +++ b/dashboard/assets/dashboard3/dashboard3.css @@ -0,0 +1,198 @@ +:root { + --bs-body-font-size: 0.875rem; + --bs-body-line-height: 1.5; +} + +html { + /* Always show scrollbar to prevent jumpiness when filtering */ + overflow-y: scroll; +} + +html, body { + padding-left: 1em; + padding-right: 1em; +} + +#banner { + font-size: larger; +} + +.banner-controls { + position: absolute; + right: 0.75rem; + top: 1.95rem; + transform: translateY(-50%); + display: flex; + gap: 0.5rem; + align-items: center; + z-index: 10; +} + +@media (max-width: 576px) { + #banner { + padding-right: 1rem; + } + + .banner-controls { + position: static; + transform: none; + margin-top: 0.5rem; + justify-content: flex-end; + } +} + +#message_box { + position: fixed; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + max-width: 90%; + width: min(40rem, 100%); + box-sizing: border-box; + opacity: 0.95; + text-align: center; + padding: 0.75rem 1rem; + z-index: 9999; +} + +#body-wrapper { + margin-top: 1em; + margin-bottom: 1em; + max-width: 100em; + margin-left: auto; + margin-right: auto; +} + +#dashboardTable { + table-layout: fixed; +} + +.tablesort { + cursor: pointer; +} + +.clickable { + cursor: pointer; +} + +.numeric { + text-align: right; +} + +.height-limit-small { + overflow: hidden; + max-height: 10em; +} + +.height-limit-big { + overflow: auto; + max-height: 30em; +} + +.job-summary td { + word-wrap: break-word; +} + +.job-ident-clicky { + border: none; + background: inherit; + color: inherit; +} + +.job-log { + /* override bootstrap 'well' */ + margin-bottom: 0px; + font-family: "Roboto Mono", monospace; + font-optical-sizing: auto; + font-weight: 400; + font-style: normal; +} + +.job-log .text-warning { + font-style: italic; +} + +.job-log .text-danger { + font-weight: 700; +} + +.job-log-line { + white-space: nowrap; +} + +.job-log-line-message { + display: block; + white-space: pre; +} + +a.job-log-line-url { + color: inherit; +} + +.job-log-paused { + overflow: auto; +} + +.job-detail-handle { + text-align: center; +} + +.job-detail-handle-active { + border-bottom: 0.2em groove mediumaquamarine; + border-radius: 0.5em; +} + +.job-progress { + height: 0.5em; + margin-bottom: 0.5em; +} + +.autoscroll-paused { + position: absolute; + top: 0px; + left: 0px; + opacity: 0.8; +} + +.extra-links { + margin-top: 3em; +} + +.pipeline { + font-size: smaller; +} + +.stick { + position: sticky; + top: 1em; + z-index: 999; +} + +.settings { + display: flex; + flex-wrap: wrap; + gap: 1em; + align-items: center; +} + +.settings > div { + flex: 0 1 auto; + min-width: max-content; +} + +@media (max-width: 576px) { + .settings { + flex-direction: column; + align-items: flex-start; + gap: 0.5em; + } + + .settings label { + white-space: nowrap; + } + + #filterBox { + max-width: 100%; + width: 100%; + } +} diff --git a/dashboard/assets/dashboard3/dashboard3.js b/dashboard/assets/dashboard3/dashboard3.js index 50633e28..90cd5d66 100644 --- a/dashboard/assets/dashboard3/dashboard3.js +++ b/dashboard/assets/dashboard3/dashboard3.js @@ -1,61 +1,68 @@ -// Generated by Haxe 3.4.2 +// Generated by Haxe 4.3.7 (function ($global) { "use strict"; -function $extend(from, fields) { - function Inherit() {} Inherit.prototype = from; var proto = new Inherit(); - for (var name in fields) proto[name] = fields[name]; - if( fields.toString !== Object.prototype.toString ) proto.toString = fields.toString; - return proto; -} -var LogLine = function() { -}; -LogLine.__name__ = true; -LogLine.prototype = { - __class__: LogLine -}; -var Job = function(ident) { - this.pendingLogLines = 0; - var _g = []; - var _g1 = 0; - while(_g1 < 62) { - var dummy = _g1++; - _g.push(0); +class LogLine { + constructor() { } - this.downloadCountBucket = _g; - this.logLines = []; - this.ident = ident; -}; -Job.__name__ = true; -Job.parseInt = function(thing) { - if(thing != null) { - try { - return Std.parseInt(thing); - } catch( error ) { - return thing; +} +class Job { + constructor(ident) { + this.pendingLogLines = 0; + let _g = []; + let _g1 = 0; + while(_g1 < 62) { + ++_g1; + _g.push(0); } - } else { - return null; + this.downloadCountBucket = _g; + this.logLines = []; + this.clusterizeRows = []; + this.ident = ident; } -}; -Job.prototype = { - fillDownloadCountBucket: function() { - var newDownloads = this.itemsDownloaded - this.lastDownloadCount; + fillDownloadCountBucket() { + let newDownloads = this.itemsDownloaded - this.lastDownloadCount; this.lastDownloadCount = this.itemsDownloaded; - var currentSecond = new Date().getSeconds(); + let currentSecond = new Date().getSeconds(); this.downloadCountBucket[currentSecond] = newDownloads; } - ,computeSpeed: function() { - var sum = 0; - var _g = 0; - var _g1 = this.downloadCountBucket; - while(_g < _g1.length) { - var count = _g1[_g]; - ++_g; - sum += count; - } + computeSpeed() { + let sum = 0; + let _g = 0; + let _g1 = this.downloadCountBucket; + while(_g < _g1.length) sum += _g1[_g++]; return sum / 60.0; } - ,consumeLogEvent: function(logEvent,maxScrollback) { - var jobData = logEvent.job_data; + getTextColor(logLine) { + if(logLine.responseCode == 200) { + return "text-success"; + } + if(logLine.responseCode >= 100 && logLine.responseCode < 200) { + return "text-primary"; + } + if(logLine.responseCode >= 201 && logLine.responseCode < 300) { + return "text-success-emphasis"; + } + if(logLine.responseCode >= 300 && logLine.responseCode < 400) { + return "text-info"; + } + if(logLine.responseCode >= 400 && logLine.responseCode < 500) { + return "text-warning"; + } + if(logLine.responseCode >= 500 && logLine.responseCode < 600) { + return "text-danger"; + } + if(logLine.isWarning) { + return "text-warning"; + } + if(logLine.isError) { + return "text-danger"; + } + if(logLine.message != null || logLine.pattern != null) { + return "text-muted"; + } + return ""; + } + consumeLogEvent(logEvent,maxScrollback) { + let jobData = logEvent.job_data; this.aborted = jobData.aborted; this.bytesDownloaded = Job.parseInt(jobData.bytes_downloaded); this.concurrency = Job.parseInt(jobData.concurrency); @@ -83,7 +90,7 @@ Job.prototype = { this.timestamp = Job.parseInt(logEvent.ts); this.url = jobData.url; this.warcSize = jobData.warc_size; - var logLine = new LogLine(); + let logLine = new LogLine(); logLine.type = logEvent.type; logLine.url = logEvent.url; logLine.timestamp = Job.parseInt(logEvent.ts); @@ -95,250 +102,255 @@ Job.prototype = { logLine.wgetCode = logEvent.wget_code; this.totalResponses = this.r1xx + this.r2xx + this.r3xx + this.r4xx + this.r5xx + this.errorCount; this.queueRemaining = this.itemsQueued - this.itemsDownloaded; - if(this.logLines.length >= maxScrollback) { - this.logLines.shift(); + if(this.logLines.length > maxScrollback) { + this.logLines = this.logLines.slice(this.logLines.length - maxScrollback); } this.fillDownloadCountBucket(); this.responsePerSecond = this.computeSpeed(); this.logLines.push(logLine); this.pendingLogLines += 1; } - ,drawPendingLogLines: function() { + drawPendingLogLines(maxScrollback) { if(this.pendingLogLines <= 0) { return; } - var logElement = window.document.getElementById("job-log-" + this.ident); - if(logElement == null) { + let scrollEl = window.document.getElementById("job-log-" + this.ident); + let contentEl = window.document.getElementById("job-log-content-" + this.ident); + if(scrollEl == null || contentEl == null) { + if(this.clusterize != null) { + this.clusterize.destroy(true); + this.clusterize = null; + } return; } - var _g = 0; - var _g1 = this.logLines.slice(-this.pendingLogLines); + if(this.clusterize != null) { + let stale = this.clusterize.scroll_elem != scrollEl || this.clusterize.content_elem != contentEl; + if(stale) { + this.clusterize.destroy(true); + this.clusterize = null; + } + } + if(this.clusterize == null) { + this.clusterizeRows = []; + let clusterizeOptions = { rows : [], scrollId : "job-log-" + this.ident, contentId : "job-log-content-" + this.ident}; + this.clusterize = new Clusterize(clusterizeOptions); + } + let _g = 0; + let _g1 = this.logLines.slice(-this.pendingLogLines); while(_g < _g1.length) { - var logLine = _g1[_g]; + let logLine = _g1[_g]; ++_g; - var logLineDiv = window.document.createElement("div"); - logLineDiv.className = "job-log-line"; - if(logLine.responseCode == 200) { - logLineDiv.classList.add("text-success"); - } else if(logLine.isWarning) { - logLineDiv.classList.add("bg-warning"); - } else if(logLine.isError) { - logLineDiv.classList.add("bg-danger"); - } else if(logLine.message != null || logLine.pattern != null) { - logLineDiv.classList.add("text-muted"); - } + let logColor = this.getTextColor(logLine); + let text = ""; if(logLine.responseCode > 0 || logLine.wgetCode != null) { - var text; if(logLine.responseCode > 0) { - text = "" + logLine.responseCode + " "; + text += "" + logLine.responseCode + " "; } else { - text = "" + logLine.wgetCode + " "; + text += "" + logLine.wgetCode + " "; } - logLineDiv.appendChild(window.document.createTextNode(text)); } if(logLine.url != null) { - var element = window.document.createElement("a"); - element.href = logLine.url; - element.textContent = logLine.url; - element.className = "job-log-line-url"; - logLineDiv.appendChild(element); + text += "" + logLine.url + ""; if(logLine.pattern != null) { - var element1 = window.document.createElement("span"); - element1.textContent = logLine.pattern; - element1.className = "text-warning"; - logLineDiv.appendChild(window.document.createTextNode(" ")); - logLineDiv.appendChild(element1); + text += " " + logLine.pattern + ""; } } else if(logLine.message != null) { - logLineDiv.textContent = logLine.message; - logLineDiv.classList.add("job-log-line-message"); + text += "" + logLine.message + ""; } - logElement.appendChild(logLineDiv); - } - var numToTrim = logElement.childElementCount - this.logLines.length; - if(numToTrim > 0) { - var _g11 = 0; - var _g2 = numToTrim; - while(_g11 < _g2) { - var dummy = _g11++; - var child = logElement.firstChild; - if(child != null) { - logElement.removeChild(child); - } + if(logColor != "") { + text = "
" + text + "
"; + } else { + text = "
" + text + "
"; } + this.clusterizeRows.push(text); + } + if(this.clusterizeRows.length > maxScrollback) { + this.clusterizeRows = this.clusterizeRows.slice(this.clusterizeRows.length - maxScrollback); } - logElement.setAttribute("data-autoscroll-dirty","true"); + this.clusterize.update(this.clusterizeRows); this.pendingLogLines = 0; } - ,attachAntiScroll: function() { - var logWindow = window.document.getElementById("job-log-" + this.ident); - if(logWindow == null) { - return; - } - if(logWindow.getAttribute("data-anti-scroll") == "attached") { - return; + enforceScrollback(maxScrollback) { + if(this.logLines.length > maxScrollback) { + this.logLines = this.logLines.slice(this.logLines.length - maxScrollback); } - logWindow.setAttribute("data-anti-scroll","attached"); - logWindow["onwheel"] = function(ev) { - if(ev.deltaY < 0 && logWindow.scrollTop == 0) { - ev.preventDefault(); - } else if(ev.deltaY > 0 && logWindow.scrollTop >= logWindow.scrollHeight - logWindow.offsetHeight) { - ev.preventDefault(); + if(this.clusterizeRows.length > maxScrollback) { + this.clusterizeRows = this.clusterizeRows.slice(this.clusterizeRows.length - maxScrollback); + if(this.clusterize != null) { + this.clusterize.update(this.clusterizeRows); } - }; - } - ,__class__: Job -}; -var Dashboard = function(hostname,maxScrollback,showNicks,drawInterval) { - if(drawInterval == null) { - drawInterval = 1000; - } - if(showNicks == null) { - showNicks = false; + } } - if(maxScrollback == null) { - maxScrollback = 500; + static parseInt(thing) { + if(thing != null) { + try { + return Std.parseInt(thing); + } catch( _g ) { + return thing; + } + } else { + return null; + } } - this.jobMap = new haxe_ds_StringMap(); - this.jobs = []; - this.angular = angular; - var _gthis = this; - this.hostname = hostname; - this.maxScrollback = maxScrollback; - this.showNicks = showNicks; - this.drawInterval = drawInterval; - this.app = this.angular.module("dashboardApp",[]); - var appConfig = ["$compileProvider",function(compileProvider) { - compileProvider.debugInfoEnabled(false); - }]; - this.app.config(appConfig); - this.app.filter("bytes",function() { - return function(num) { - var _g = 0; - var _g1 = ["B","KiB","MiB","GiB"]; - while(_g < _g1.length) { - var unit = _g1[_g]; - ++_g; +} +class Dashboard { + constructor(hostname,maxScrollback,showNicks,drawInterval) { + if(drawInterval == null) { + drawInterval = 1000; + } + if(showNicks == null) { + showNicks = false; + } + if(maxScrollback == null) { + maxScrollback = 500; + } + this.jobMap = new haxe_ds_StringMap(); + this.jobs = []; + this.angular = angular; + this.hostname = hostname; + this.maxScrollback = maxScrollback; + this.showNicks = showNicks; + this.drawInterval = drawInterval; + this.app = this.angular.module("dashboardApp",[]); + let appConfig = ["$compileProvider",function(compileProvider) { + compileProvider.debugInfoEnabled(false); + }]; + this.app.config(appConfig); + this.app.filter("bytes",function() { + return function(num) { if(num < 1024 && num > -1024) { num = Math.round(num * 10) / 10; - return "" + num + " " + unit; + return "" + num + " " + "B"; } num /= 1024.0; - } - num = Math.round(num * 10) / 10; - return "" + num + " TiB"; - }; - }); - var controllerArgs = ["$scope",function(scope) { - scope.jobs = _gthis.jobs; - scope.filterQuery = ""; - scope.hideDetails = false; - scope.paused = false; - scope.sortParam = "startedAt"; - scope.showNicks = showNicks; - scope.drawInterval = drawInterval; - _gthis.dashboardControllerScopeApply = Reflect.field(scope,"$apply").bind(scope); - scope.filterOperator = function(job) { - var query = scope.filterQuery; - if(scope.showNicks) { - if(!(StringTools.startsWith(job.ident,query) || job.url.indexOf(query) != -1)) { - return job.startedBy.toLowerCase().indexOf(query.toLowerCase()) != -1; + if(num < 1024 && num > -1024) { + num = Math.round(num * 10) / 10; + return "" + num + " " + "KiB"; + } + num /= 1024.0; + if(num < 1024 && num > -1024) { + num = Math.round(num * 10) / 10; + return "" + num + " " + "MiB"; + } + num /= 1024.0; + if(num < 1024 && num > -1024) { + num = Math.round(num * 10) / 10; + return "" + num + " " + "GiB"; + } + num /= 1024.0; + num = Math.round(num * 10) / 10; + return "" + num + " TiB"; + }; + }); + let _gthis = this; + let controllerArgs = ["$scope",function(scope) { + scope.jobs = _gthis.jobs; + scope.filterQuery = ""; + scope.hideDetails = false; + scope.paused = false; + scope.sortParam = "startedAt"; + scope.showNicks = showNicks; + scope.drawInterval = drawInterval; + scope.currentPage = 1; + scope.pageSize = 20; + scope.totalPages = 1; + _gthis.dashboardControllerScopeApply = Reflect.field(scope,"$apply").bind(scope); + scope.filterOperator = function(job) { + let query = scope.filterQuery; + if(scope.showNicks) { + if(!(job.ident.startsWith(query) || job.url.indexOf(query) != -1)) { + return job.startedBy.toLowerCase().indexOf(query.toLowerCase()) != -1; + } else { + return true; + } + } else if(!job.ident.startsWith(query)) { + return job.url.indexOf(query) != -1; } else { return true; } - } else if(!StringTools.startsWith(job.ident,query)) { - return job.url.indexOf(query) != -1; - } else { - return true; - } - }; - _gthis.dashboardControllerScope = scope; - scope.applyFilterQuery = function(query1) { - scope.filterQuery = query1; - }; - }]; - this.app.controller("DashboardController",controllerArgs); -}; -Dashboard.__name__ = true; -Dashboard.getQueryArgs = function() { - var query = window.location.search; - var items = StringTools.replace(query,"?","").split("&"); - var args = new haxe_ds_StringMap(); - var _g = 0; - while(_g < items.length) { - var item = items[_g]; - ++_g; - var pairs = item.split("="); - var key = pairs[0]; - var value = pairs[1]; - if(__map_reserved[key] != null) { - args.setReserved(key,value); - } else { - args.h[key] = value; - } - } - return args; -}; -Dashboard.main = function() { - var args = Dashboard.getQueryArgs(); - var hostname; - var maxScrollback = 20; - var showNicks = __map_reserved["showNicks"] != null ? args.existsReserved("showNicks") : args.h.hasOwnProperty("showNicks"); - if(__map_reserved["host"] != null ? args.existsReserved("host") : args.h.hasOwnProperty("host")) { - hostname = __map_reserved["host"] != null ? args.getReserved("host") : args.h["host"]; - } else { - hostname = window.location.hostname; + }; + _gthis.dashboardControllerScope = scope; + scope.applyFilterQuery = function(query) { + scope.filterQuery = query; + }; + scope.$watchGroup(["filterQuery","jobs.length"],function(newVals,oldVals) { + let filterQuery = newVals[0]; + let maxScrollback = 500; + if(filterQuery == null || filterQuery.trim() == "") { + maxScrollback = 50; + } + _gthis.changeMaxScrollback(maxScrollback); + let _g = 0; + let _g1 = _gthis.jobs; + while(_g < _g1.length) _g1[_g++].enforceScrollback(_gthis.maxScrollback); + let _this = scope.jobs; + let f = scope.filterOperator; + let _g2 = []; + let _g3 = 0; + while(_g3 < _this.length) { + let v = _this[_g3]; + ++_g3; + if(f(v)) { + _g2.push(v); + } + } + scope.totalPages = Math.max(1,Math.ceil(_g2.length / scope.pageSize)) | 0; + if(oldVals != null && filterQuery != oldVals[0]) { + scope.currentPage = 1; + } else if(scope.currentPage > scope.totalPages) { + scope.currentPage = scope.totalPages; + } + }); + scope.setPage = function(page) { + if(page >= 1 && page <= scope.totalPages) { + scope.currentPage = page; + } + }; + }]; + this.app.controller("DashboardController",controllerArgs); } - if(window.navigator.userAgent.indexOf("Mobi") == -1) { - maxScrollback = 500; + changeMaxScrollback(maxScrollback) { + this.maxScrollback = maxScrollback; } - var dashboard = new Dashboard(hostname,maxScrollback,showNicks); - dashboard.run(); -}; -Dashboard.prototype = { - run: function() { + run() { this.loadRecentLogs(); } - ,loadRecentLogs: function() { - var _gthis = this; - var request = new XMLHttpRequest(); + loadRecentLogs() { + let request = new XMLHttpRequest(); + let _gthis = this; request.onerror = function(event) { _gthis.showError("Unable to load dashboard. Reload the page?"); }; - request.onload = function(event1) { + request.onload = function(event) { if(request.status != 200) { _gthis.showError("The server didn't respond correctly: " + request.status + " " + request.statusText); return; } _gthis.showError(null); - var doc = JSON.parse(request.responseText); - var _g = 0; - while(_g < doc.length) { - var logEvent = doc[_g]; - ++_g; - _gthis.processLogEvent(logEvent); - } + let doc = JSON.parse(request.responseText); + let _g = 0; + while(_g < doc.length) _gthis.processLogEvent(doc[_g++]); _gthis.scheduleDraw(); _gthis.openWebSocket(); }; - var cacheBustValue = new Date().getTime(); + let cacheBustValue = new Date().getTime(); request.open("GET","/logs/recent?cb=" + cacheBustValue); request.setRequestHeader("Accept","application/json"); request.send(""); } - ,openWebSocket: function() { - var _gthis = this; + openWebSocket() { if(this.websocket != null) { return; } - var wsProto = window.location.protocol == "https:" ? "wss:" : "ws:"; + let wsProto = $global.location.protocol == "https:" ? "wss:" : "ws:"; this.websocket = new WebSocket("" + wsProto + "//" + this.hostname + ":4568/stream"); + let _gthis = this; this.websocket.onmessage = function(message) { _gthis.showError(null); - var doc = JSON.parse(message.data); + let doc = JSON.parse(message.data); _gthis.processLogEvent(doc); }; - this.websocket.onclose = function(message1) { + this.websocket.onclose = function(message) { if(_gthis.websocket == null) { return; } @@ -350,18 +362,18 @@ Dashboard.prototype = { }; this.websocket.onerror = this.websocket.onclose; } - ,scheduleDraw: function(delayMS) { + scheduleDraw(delayMS) { if(delayMS == null) { delayMS = 1000; } - var _gthis = this; + let _gthis = this; this.drawTimerHandle = setTimeout(function() { - var delay = _gthis.dashboardControllerScope.drawInterval; + let delay = _gthis.dashboardControllerScope.drawInterval; if(!window.document.hidden && !_gthis.dashboardControllerScope.paused) { - var beforeDate = new Date(); + let beforeDate = new Date(); _gthis.redraw(); - var afterDate = new Date(); - var difference = afterDate.getTime() - beforeDate.getTime(); + let afterDate = new Date(); + let difference = afterDate.getTime() - beforeDate.getTime(); if(difference > 10) { delay += difference * 2; delay = Math.min(delay,10000); @@ -370,32 +382,21 @@ Dashboard.prototype = { _gthis.scheduleDraw(delay); },delayMS); } - ,processLogEvent: function(logEvent) { - var job; - var ident = logEvent.job_data.ident; - var _this = this.jobMap; - if(!(__map_reserved[ident] != null ? _this.existsReserved(ident) : _this.h.hasOwnProperty(ident))) { + processLogEvent(logEvent) { + let job; + let ident = logEvent.job_data.ident; + if(!Object.prototype.hasOwnProperty.call(this.jobMap.h,ident)) { job = new Job(ident); - var _this1 = this.jobMap; - if(__map_reserved[ident] != null) { - _this1.setReserved(ident,job); - } else { - _this1.h[ident] = job; - } + this.jobMap.h[ident] = job; this.jobs.push(job); - console.log("Load job " + ident); + console.log("Dashboard.hx:566:","Load job " + ident); } else { - var _this2 = this.jobMap; - if(__map_reserved[ident] != null) { - job = _this2.getReserved(ident); - } else { - job = _this2.h[ident]; - } + job = this.jobMap.h[ident]; } job.consumeLogEvent(logEvent,this.maxScrollback); } - ,showError: function(message) { - var element = window.document.getElementById("message_box"); + showError(message) { + let element = window.document.getElementById("message_box"); if(message != null) { element.style.display = "block"; element.innerText = message; @@ -403,356 +404,97 @@ Dashboard.prototype = { element.style.display = "none"; } } - ,redraw: function() { + redraw() { this.dashboardControllerScopeApply(); - var _g = 0; - var _g1 = this.jobs; + let _g = 0; + let _g1 = this.jobs; while(_g < _g1.length) { - var job = _g1[_g]; + let job = _g1[_g]; ++_g; if(!job.logPaused) { - job.drawPendingLogLines(); + job.drawPendingLogLines(this.maxScrollback); } } this.scrollLogsToBottom(); } - ,scrollLogsToBottom: function() { - var nodes = window.document.querySelectorAll("[data-autoscroll-dirty].autoscroll"); - var pending = []; - var _g = 0; - while(_g < nodes.length) { - var node = nodes[_g]; + scrollLogsToBottom() { + let _g = 0; + let _g1 = this.jobs; + while(_g < _g1.length) { + let job = _g1[_g]; ++_g; - var element = js_Boot.__cast(node , HTMLElement); - element.removeAttribute("data-autoscroll-dirty"); - pending.push(element); - } - var _g1 = 0; - while(_g1 < pending.length) { - var element1 = pending[_g1]; - ++_g1; - element1.scrollTop = 99999; + if(!job.logPaused && job.clusterize != null) { + let scrollEl = window.document.getElementById("job-log-" + job.ident); + if(scrollEl != null) { + scrollEl.scrollTop = scrollEl.scrollHeight; + } + } } } - ,__class__: Dashboard -}; -var HxOverrides = function() { }; -HxOverrides.__name__ = true; -HxOverrides.cca = function(s,index) { - var x = s.charCodeAt(index); - if(x != x) { - return undefined; - } - return x; -}; -HxOverrides.substr = function(s,pos,len) { - if(len == null) { - len = s.length; - } else if(len < 0) { - if(pos == 0) { - len = s.length + len; - } else { - return ""; + static getQueryArgs() { + let query = $global.location.search; + let items = StringTools.replace(query,"?","").split("&"); + let args = new haxe_ds_StringMap(); + let _g = 0; + while(_g < items.length) { + let pairs = items[_g++].split("="); + args.h[pairs[0]] = pairs[1]; } + return args; } - return s.substr(pos,len); -}; -Math.__name__ = true; -var Reflect = function() { }; -Reflect.__name__ = true; -Reflect.field = function(o,field) { - try { - return o[field]; - } catch( e ) { - return null; - } -}; -var Std = function() { }; -Std.__name__ = true; -Std.string = function(s) { - return js_Boot.__string_rec(s,""); -}; -Std.parseInt = function(x) { - var v = parseInt(x,10); - if(v == 0 && (HxOverrides.cca(x,1) == 120 || HxOverrides.cca(x,1) == 88)) { - v = parseInt(x); - } - if(isNaN(v)) { - return null; - } - return v; -}; -var StringTools = function() { }; -StringTools.__name__ = true; -StringTools.startsWith = function(s,start) { - if(s.length >= start.length) { - return HxOverrides.substr(s,0,start.length) == start; - } else { - return false; - } -}; -StringTools.replace = function(s,sub,by) { - return s.split(sub).join(by); -}; -var haxe_IMap = function() { }; -haxe_IMap.__name__ = true; -var haxe_ds_StringMap = function() { - this.h = { }; -}; -haxe_ds_StringMap.__name__ = true; -haxe_ds_StringMap.__interfaces__ = [haxe_IMap]; -haxe_ds_StringMap.prototype = { - setReserved: function(key,value) { - if(this.rh == null) { - this.rh = { }; - } - this.rh["$" + key] = value; - } - ,getReserved: function(key) { - if(this.rh == null) { - return null; + static main() { + let args = Dashboard.getQueryArgs(); + let hostname; + let showNicks = Object.prototype.hasOwnProperty.call(args.h,"showNicks"); + if(Object.prototype.hasOwnProperty.call(args.h,"host")) { + hostname = args.h["host"]; } else { - return this.rh["$" + key]; - } - } - ,existsReserved: function(key) { - if(this.rh == null) { - return false; + hostname = $global.location.hostname; } - return this.rh.hasOwnProperty("$" + key); - } - ,__class__: haxe_ds_StringMap -}; -var js__$Boot_HaxeError = function(val) { - Error.call(this); - this.val = val; - this.message = String(val); - if(Error.captureStackTrace) { - Error.captureStackTrace(this,js__$Boot_HaxeError); + new Dashboard(hostname,50,showNicks).run(); } -}; -js__$Boot_HaxeError.__name__ = true; -js__$Boot_HaxeError.wrap = function(val) { - if((val instanceof Error)) { - return val; - } else { - return new js__$Boot_HaxeError(val); - } -}; -js__$Boot_HaxeError.__super__ = Error; -js__$Boot_HaxeError.prototype = $extend(Error.prototype,{ - __class__: js__$Boot_HaxeError -}); -var js_Boot = function() { }; -js_Boot.__name__ = true; -js_Boot.getClass = function(o) { - if((o instanceof Array) && o.__enum__ == null) { - return Array; - } else { - var cl = o.__class__; - if(cl != null) { - return cl; - } - var name = js_Boot.__nativeClassName(o); - if(name != null) { - return js_Boot.__resolveNativeClass(name); - } - return null; - } -}; -js_Boot.__string_rec = function(o,s) { - if(o == null) { - return "null"; - } - if(s.length >= 5) { - return "<...>"; - } - var t = typeof(o); - if(t == "function" && (o.__name__ || o.__ename__)) { - t = "object"; - } - switch(t) { - case "function": - return ""; - case "object": - if(o instanceof Array) { - if(o.__enum__) { - if(o.length == 2) { - return o[0]; - } - var str = o[0] + "("; - s += "\t"; - var _g1 = 2; - var _g = o.length; - while(_g1 < _g) { - var i = _g1++; - if(i != 2) { - str += "," + js_Boot.__string_rec(o[i],s); - } else { - str += js_Boot.__string_rec(o[i],s); - } - } - return str + ")"; - } - var l = o.length; - var i1; - var str1 = "["; - s += "\t"; - var _g11 = 0; - var _g2 = l; - while(_g11 < _g2) { - var i2 = _g11++; - str1 += (i2 > 0 ? "," : "") + js_Boot.__string_rec(o[i2],s); - } - str1 += "]"; - return str1; - } - var tostr; +} +class Reflect { + static field(o,field) { try { - tostr = o.toString; - } catch( e ) { - return "???"; - } - if(tostr != null && tostr != Object.toString && typeof(tostr) == "function") { - var s2 = o.toString(); - if(s2 != "[object Object]") { - return s2; - } + return o[field]; + } catch( _g ) { + return null; } - var k = null; - var str2 = "{\n"; - s += "\t"; - var hasp = o.hasOwnProperty != null; - for( var k in o ) { - if(hasp && !o.hasOwnProperty(k)) { - continue; - } - if(k == "prototype" || k == "__class__" || k == "__super__" || k == "__interfaces__" || k == "__properties__") { - continue; - } - if(str2.length != 2) { - str2 += ", \n"; - } - str2 += s + k + " : " + js_Boot.__string_rec(o[k],s); - } - s = s.substring(1); - str2 += "\n" + s + "}"; - return str2; - case "string": - return o; - default: - return String(o); - } -}; -js_Boot.__interfLoop = function(cc,cl) { - if(cc == null) { - return false; } - if(cc == cl) { - return true; - } - var intf = cc.__interfaces__; - if(intf != null) { - var _g1 = 0; - var _g = intf.length; - while(_g1 < _g) { - var i = _g1++; - var i1 = intf[i]; - if(i1 == cl || js_Boot.__interfLoop(i1,cl)) { - return true; - } +} +class Std { + static parseInt(x) { + let v = parseInt(x); + if(isNaN(v)) { + return null; } + return v; } - return js_Boot.__interfLoop(cc.__super__,cl); -}; -js_Boot.__instanceof = function(o,cl) { - if(cl == null) { - return false; +} +class StringTools { + static replace(s,sub,by) { + return s.split(sub).join(by); } - switch(cl) { - case Array: - if((o instanceof Array)) { - return o.__enum__ == null; - } else { - return false; - } - break; - case Bool: - return typeof(o) == "boolean"; - case Dynamic: - return true; - case Float: - return typeof(o) == "number"; - case Int: - if(typeof(o) == "number") { - return (o|0) === o; - } else { - return false; - } - break; - case String: - return typeof(o) == "string"; - default: - if(o != null) { - if(typeof(cl) == "function") { - if(o instanceof cl) { - return true; - } - if(js_Boot.__interfLoop(js_Boot.getClass(o),cl)) { - return true; - } - } else if(typeof(cl) == "object" && js_Boot.__isNativeObj(cl)) { - if(o instanceof cl) { - return true; - } - } - } else { - return false; - } - if(cl == Class ? o.__name__ != null : false) { - return true; - } - if(cl == Enum ? o.__ename__ != null : false) { - return true; - } - return o.__enum__ == cl; +} +class haxe_ds_StringMap { + constructor() { + this.h = Object.create(null); } -}; -js_Boot.__cast = function(o,t) { - if(js_Boot.__instanceof(o,t)) { - return o; - } else { - throw new js__$Boot_HaxeError("Cannot cast " + Std.string(o) + " to " + Std.string(t)); +} +class haxe_iterators_ArrayIterator { + constructor(array) { + this.current = 0; + this.array = array; } -}; -js_Boot.__nativeClassName = function(o) { - var name = js_Boot.__toStr.call(o).slice(8,-1); - if(name == "Object" || name == "Function" || name == "Math" || name == "JSON") { - return null; + hasNext() { + return this.current < this.array.length; } - return name; -}; -js_Boot.__isNativeObj = function(o) { - return js_Boot.__nativeClassName(o) != null; -}; -js_Boot.__resolveNativeClass = function(name) { - return $global[name]; -}; -String.prototype.__class__ = String; -String.__name__ = true; -Array.__name__ = true; -Date.prototype.__class__ = Date; -Date.__name__ = ["Date"]; -var Int = { __name__ : ["Int"]}; -var Dynamic = { __name__ : ["Dynamic"]}; -var Float = Number; -Float.__name__ = ["Float"]; -var Bool = Boolean; -Bool.__ename__ = ["Bool"]; -var Class = { __name__ : ["Class"]}; -var Enum = { }; -var __map_reserved = {} -Job.isSafari = window.navigator.userAgent.indexOf("Safari") != -1; -js_Boot.__toStr = ({ }).toString; + next() { + return this.array[this.current++]; + } +} +{ +} Dashboard.main(); })(typeof window != "undefined" ? window : typeof global != "undefined" ? global : typeof self != "undefined" ? self : this); diff --git a/dashboard/dashboard3.html b/dashboard/dashboard3.html index a431ff01..71f35a1f 100644 --- a/dashboard/dashboard3.html +++ b/dashboard/dashboard3.html @@ -10,196 +10,125 @@ - + + + + + + + + + ArchiveBot dashboard 3.0 -
- - -
-

- This page shows all of the crawls that ArchiveBot is currently running. -

-

- To clear all finished jobs, reload the page. -

-

- If your adblocker is enabled for this domain, you will see slower performance, and some URLs will not be displayed. -

-

- To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. -

-

- These ignore sets are available for crawls. The global ignore set automatically applies to all crawls. -

-

- GitHub: ArchiveBot. -

-

+

+ + +
+

+ This page shows all of the crawls that ArchiveBot is currently running. +

+

+ To clear all finished jobs, reload the page. +

+

+ If your adblocker is enabled for this domain, you will see slower performance, and some URLs will not be displayed. +

+

+ To use ArchiveBot, drop by #archivebot on hackint. Issue commands by typing them into the channel. You will need channel operator (@) or voice (+) status to issue archiving jobs; just ask for help or leave a message with the website you want to archive. +

+

+ These ignore sets are available for crawls. The global ignore set automatically applies to all crawls. +

+

+ GitHub: ArchiveBot. +

+ +
+ +
+ Loading... +
+ +
+
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+
+
+ +
-
- Loading... -
+
-
- - - -
- -
- - - - - - - - -
- - +
- - - - - - - + + + + + + + - +
URLStarted byStartedFetchedSizeRemainIdentURLStarted byStartedFetchedSizeRemainIdent
!{{ :: job.depth == 'shallow' ? 'ao' : 'a' }}
-
-
+
+
+
+ + +

with @@ -223,30 +156,30 @@ {{ job.suppressIgnoreReports ? 'Hiding' : 'Showing' }} ignores.

-
+
-
-
-
-
-
- +
- +
@@ -297,32 +230,25 @@

-
-
+
-
Paused
+ ng-mouseover="job.logPaused = true" ng-mouseleave="job.logPaused = false"> +
+
+
+ Paused +
{{:: job.attachAntiScroll() }}
- -
-
- -
-
- -
-
-
Done
- +