This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 47dd78d8381cadc99b65d347327e9dd9f2a6ae22 Author: Surya Hebbar <[email protected]> AuthorDate: Mon Jan 8 18:23:13 2024 +0530 IMPALA-12688: Support JSON profile imports in webUI A button has been added to import query profiles on the queries page. The button opens a system dialog to browse only "*.json" files from client's local file system. This allows multiple JSON query profiles to be imported. JSON profiles are read from file system using FileReader API and then these imported query profiles are stored in a NoSQL database using browser's IndexedDB API. If an error is encountered while parsing JSON profile, it is displayed on the queries page. IndexedDB is a large-scale NoSQL database system, that can be read and written to asynchronously. Hence, the file size and number of JSON profiles is only limited by the storage available on the client system. The storage limit is browser dependent, in chromium based browsers upto 80% of local storage may be used by IndexedDB. The profile is parsed for the following attributes, which will be displayed on the queries page. Number of imported queries is also displayed. - Query ID - User - Default Db - Query Type - Start Time - End Time - Bytes Read - Bytes Sent - Query State - No. of rows fetched - Resource Pool - Query Statement After importing or deleting query profiles, page will scroll back to the imported query profiles section. The queries are sorted based on descending order of the "Start Time". A button is displayed beside the import button to easily clear all the imported queries at once. On opening any of the imported queries, its query timeline will be displayed. Currently the following tabs are supported for imported queries. - Query Statement - Query Timeline - Query Text Plan Syntax highlighting on query statement tab has also been supported. Change-Id: Ife6eb59bf2030fd19fc92aaf134eb51c609e04d0 Reviewed-on: http://gerrit.cloudera.org:8080/20867 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- www/queries.tmpl | 191 ++++++++++++++++++++- www/query_plan_text.tmpl | 42 ++++- www/query_stmt.tmpl | 49 +++++- www/query_timeline.tmpl | 92 +++++++--- .../query_timeline/fragment_metrics_diagram.js | 14 +- .../query_timeline/host_utilization_diagram.js | 11 +- 6 files changed, 363 insertions(+), 36 deletions(-) diff --git a/www/queries.tmpl b/www/queries.tmpl index 95a763a2f..fc3097f65 100644 --- a/www/queries.tmpl +++ b/www/queries.tmpl @@ -17,9 +17,17 @@ specific language governing permissions and limitations under the License. --> {{> www/common-header.tmpl }} + +<style> +#json_profile_chooser { + display: none; +} +</style> + <h2>Queries</h2> -<p class="lead">This page lists all running queries, plus any completed queries that are -archived in memory. The size of that archive is controlled with the +<p class="lead">This page lists all running queries plus completed queries +that are archived in memory, and imported query profiles. +The size of that archive is controlled with the <samp>--query_log_size</samp> command line parameter.</p> <p>The length of the statements are controlled with the <samp>--query_stmt_size</samp> command line parameter.</p> @@ -205,6 +213,37 @@ command line parameter.</p> {{/completed_queries}} </table> +<h3 id="imported_queries_header"> + 0 Imported Query Profiles + <sup><a href='#' data-toggle="tooltip" title="These are locally stored queries parsed from JSON query profiles.">[?]</a></sup> + <input id="json_profile_chooser" type="file" accept=".json" onchange="uploadProfile();" multiple/> + <label for="json_profile_chooser"> + <span class="btn btn-primary">Import JSON Profile</span> + </label> + <input id="clear_profiles_button" type="button" class="btn btn-primary" title="Clear All" onclick="clearProfiles();" value="X"/> +</h3> + +<table id="imported_queries_table" class='table table-hover table-border'> + <tr> + <th title="{{tips_query_id}}">Query ID</th> + <th title="{{tips_user}}">User</th> + <th title="{{tips_default_db}}">Default Db</th> + <th title="{{tips_query_type}}">Query Type</th> + <th title="{{tips_start_time}}">Start Time</th> + <th title="{{tips_end_time}}">End Time</th> + <th> + <span title="{{tips_bytes_read}}">Bytes Read</span> + <hr style="margin-top:0px;margin-bottom:0px;"/> + <span title="{{tips_bytes_sent}}">Bytes Sent</span> + </th> + <th title="{{tips_state}}">State</th> + <th title="{{tips_rows_fetched}}"># rows fetched</th> + <th title="{{tips_resource_pool}}">Resource Pool</th> + <th title="{{tips_statement}}">Statement</th> + <th>Delete</th> + </tr> +</table> + <h3>Query Locations</h3> <table class='table table-hover table-bordered'> @@ -221,5 +260,153 @@ command line parameter.</p> </tr> {{/query_locations}} </table> +<script> + var dbOpenReq = indexedDB.open("imported_queries"); + var db; + + function insertRowVal(row, val) { + row.insertCell().innerHTML = val; + } + + function setScrollReload() { + localStorage.setItem("imported", "T"); + window.location.reload(); + } + + function deleteProfile(el) { + var row = el.parentElement.parentElement; + var profileStore = db.transaction("profiles", "readwrite").objectStore("profiles"); + profileStore.delete(row.cells[0].textContent).onsuccess = () => { + setScrollReload(); + }; + } + + function clearProfiles() { + db.transaction("profiles", "readwrite").objectStore("profiles").clear(); + setScrollReload(); + } + + function uploadProfile() { + var uploadCount = 0; + for (var i = 0; i < json_profile_chooser.files.length; i++) { + var fileReader = new FileReader(); + fileReader.readAsText(json_profile_chooser.files[i]); + fileReader.onload = (e) => { + try { + var profile = JSON.parse(e.target.result).contents; + var val = profile.profile_name; + var query = {}; + query.id = val.substring(val.indexOf("=") + 1, val.length - 1); + query.user = profile.child_profiles[0].info_strings + .find(({key}) => key === "User").value; + query.default_db = profile.child_profiles[0].info_strings + .find(({key}) => key === "Default Db").value; + query.type = profile.child_profiles[0].info_strings + .find(({key}) => key === "Query Type").value; + query.start_time = profile.child_profiles[0].info_strings + .find(({key}) => key === "Start Time").value; + query.end_time = profile.child_profiles[0].info_strings + .find(({key}) => key === "End Time").value; + query.bytes_read = profile.child_profiles[2].counters + .find(({counter_name}) => counter_name === "TotalBytesRead").value; + query.bytes_read = getReadableSize(query.bytes_read, 2); + query.bytes_sent = profile.child_profiles[2].counters + .find(({counter_name}) => counter_name === "TotalBytesSent").value; + query.bytes_sent = getReadableSize(query.bytes_sent, 2); + query.state = profile.child_profiles[0].info_strings + .find(({key}) => key === "Query State").value; + query.rows_fetched = profile.child_profiles[1].counters + .find(({counter_name}) => counter_name === "NumRowsFetched").value; + query.resource_pool = profile.child_profiles[0].info_strings + .find(({key}) => key === "Request Pool").value; + query.statement = profile.child_profiles[0].info_strings + .find(({key}) => key === "Sql Statement").value; + if (query.statement.length > 250) { + query.statement = query.statement.substring(0, 250) + "..."; + } + query.profile = profile; + + var profileStore = db.transaction("profiles", "readwrite").objectStore("profiles"); + profileStore.put(query).onsuccess = () => { + uploadCount++; + if (uploadCount == json_profile_chooser.files.length) { + setScrollReload(); + } + }; + } catch (err) { + var alertMessage = document.createElement("span"); + alertMessage.className = "alert-sm alert-danger"; + alertMessage.textContent = "Error parsing some JSON profiles"; + setTimeout(setScrollReload, 1500); + imported_queries_header.appendChild(alertMessage); + console.log(err); + } + }; + } + } + + dbOpenReq.onupgradeneeded = (e) => { + db = e.target.result; + var profileStore = db.createObjectStore("profiles", { keyPath : "id" }); + + profileStore.createIndex("start_time", "start_time"); + }; + + dbOpenReq.onsuccess = (e) => { + db = e.target.result; + db.onerror = (e) => { + console.log("IndexedDB error"); + console.log(e); + } + + var profileStore = db.transaction("profiles", "readonly").objectStore("profiles"); + var countReq = profileStore.count(); + + countReq.onsuccess = () => { + if (countReq.result) { + imported_queries_header.innerHTML = imported_queries_header.innerHTML + .replace("0", String(countReq.result)); + } else { + clear_profiles_button.remove(); + } + }; + + profileStore.index("start_time").openCursor(null, "prev").onsuccess = (e) => { + var cursor = e.target.result; + if (cursor) { + var query = cursor.value; + var row = imported_queries_table.insertRow(); + var query_link = document.createElement("a"); + query_link.href = `/query_timeline?query_id=${query.id}&imported=true`; + query_link.innerHTML = query.id; + insertRowVal(row, query_link.outerHTML); + insertRowVal(row, query.user); + insertRowVal(row, query.default_db); + insertRowVal(row, query.type); + insertRowVal(row, query.start_time); + insertRowVal(row, query.end_time); + insertRowVal(row, `${query.bytes_read} + <hr style="margin-top:0px;margin-bottom:0px;"/>${query.bytes_sent}`); + insertRowVal(row, query.state); + insertRowVal(row, query.rows_fetched); + insertRowVal(row, query.resource_pool); + insertRowVal(row, query.statement); + var deleteButton = document.createElement("input"); + deleteButton.type = "button"; + deleteButton.value = "X"; + deleteButton.className = "btn btn-primary"; + deleteButton.setAttribute("onclick", "deleteProfile(this);"); + insertRowVal(row, deleteButton.outerHTML); + cursor.continue(); + } + }; + }; + window.onload = () => { + if (localStorage.getItem("imported") == "T") { + imported_queries_header.scrollIntoView({ behavior : "instant", block : "start" }); + localStorage.setItem("imported", "F"); + } + }; +</script> {{> www/common-footer.tmpl }} diff --git a/www/query_plan_text.tmpl b/www/query_plan_text.tmpl index 3f922f0fe..add1e4448 100644 --- a/www/query_plan_text.tmpl +++ b/www/query_plan_text.tmpl @@ -21,10 +21,50 @@ under the License. {{> www/query_detail_tabs.tmpl }} -<pre>{{plan}}</pre> +<pre id="query_plan">{{plan}}</pre> <script> $("#plan-text-tab").addClass("active"); + + +var dbOpenReq = indexedDB.open("imported_queries"); +var db; + +var supported_tabs = ["Query", "Timeline", "Text plan"]; + +if (window.location.search.includes("imported")) { + var alertMessage = document.getElementsByClassName("alert alert-danger")[0]; + if (alertMessage) { + alertMessage.remove(); + } + var nav_links = document.getElementsByClassName("nav nav-tabs")[0]; + nav_links = nav_links.getElementsByClassName("nav-link"); + for (var i = 0; i < nav_links.length;) { + if (supported_tabs.includes(nav_links[i].textContent)) { + nav_links[i].href = `${nav_links[i].href}&imported=true`; + i++; + } else { + nav_links[i].parentElement.remove(); + } + } + + dbOpenReq.onsuccess = (e) => { + db = e.target.result; + db.onerror = (e) => { + console.log("IndexedDB error"); + console.log(e); + } + var profileStore = db.transaction("profiles", "readonly").objectStore("profiles"); + var query = {}; + query.id = window.location.search; + query.id = query.id.substring(10, query.id.indexOf("&")); + profileStore.get(query.id).onsuccess = (e) => { + query = e.target.result; + query_plan.textContent = query.profile.child_profiles[0].info_strings + .find(({key}) => key === "Plan").value; + }; + }; +} </script> {{> www/common-footer.tmpl }} diff --git a/www/query_stmt.tmpl b/www/query_stmt.tmpl index a5886ff56..188b40c89 100644 --- a/www/query_stmt.tmpl +++ b/www/query_stmt.tmpl @@ -30,10 +30,57 @@ under the License. <script src="{{ __common__.host-url }}/www/highlight/highlight.pack.js"></script> <script>hljs.initHighlightingOnLoad();</script> -{{?stmt}}<pre class="code"><code>{{stmt}}</code></pre>{{/stmt}} +<span id="tab_body">{{?stmt}}<pre class="code"><code>{{stmt}}</code></pre>{{/stmt}}</span> <script> $("#stmt-tab").addClass("active"); + +var dbOpenReq = indexedDB.open("imported_queries"); +var db; + +var supported_tabs = ["Query", "Timeline", "Text plan"]; + +if (window.location.search.includes("imported")) { + var alertMessage = document.getElementsByClassName("alert alert-danger")[0]; + if (alertMessage) { + alertMessage.remove(); + } + var nav_links = document.getElementsByClassName("nav nav-tabs")[0]; + nav_links = nav_links.getElementsByClassName("nav-link"); + for (var i = 0; i < nav_links.length;) { + if (supported_tabs.includes(nav_links[i].textContent)) { + nav_links[i].href = `${nav_links[i].href}&imported=true`; + i++; + } else { + nav_links[i].parentElement.remove(); + } + } + + dbOpenReq.onsuccess = (e) => { + db = e.target.result; + db.onerror = (e) => { + console.log("IndexedDB error"); + console.log(e); + } + var profileStore = db.transaction("profiles", "readonly").objectStore("profiles"); + var query = {}; + query.id = window.location.search; + query.id = query.id.substring(10, query.id.indexOf("&")); + profileStore.get(query.id).onsuccess = (e) => { + query = e.target.result; + var sql_query = query.profile.child_profiles[0].info_strings + .find(({key}) => key === "Sql Statement").value; + var sql_stmt_body = document.createElement("pre"); + sql_stmt_body.className = "code"; + var sql_code = sql_stmt_body.appendChild(document.createElement("code")) + sql_code.innerHTML = sql_query; + tab_body.innerHTML = ""; + tab_body.appendChild(sql_stmt_body); + hljs.highlightBlock(sql_stmt_body); + }; + }; +} + </script> {{> www/common-footer.tmpl }} diff --git a/www/query_timeline.tmpl b/www/query_timeline.tmpl index 4de641c69..8288e5349 100644 --- a/www/query_timeline.tmpl +++ b/www/query_timeline.tmpl @@ -162,37 +162,83 @@ $("#plan-timing-tab").addClass("active"); import {renderTimingDiagram, collectFragmentEventsFromProfile, ntics, set_ntics} from "./www/scripts/query_timeline/fragment_diagram.js"; -import {collectUtilizationFromProfile, toogleUtilizationVisibility} from - "/www/scripts/query_timeline/host_utilization_diagram.js"; -import {collectFragmentMetricsFromProfile} from +import {collectUtilizationFromProfile, toogleUtilizationVisibility, + destroyUtilizationChart} + from "/www/scripts/query_timeline/host_utilization_diagram.js"; +import {collectFragmentMetricsFromProfile, closeFragmentMetricsChart} from "/www/scripts/query_timeline/fragment_metrics_diagram.js"; -import {profile, set_profile, diagram_width, set_diagram_width, border_stroke_width, - resizeHorizontalAll, maxts} from "/www/scripts/query_timeline/global_members.js"; +import {profile, set_profile, maxts, set_maxts, diagram_width, set_diagram_width, + border_stroke_width, resizeHorizontalAll} + from "/www/scripts/query_timeline/global_members.js"; var chart_export_style; var last_maxts; -window.onload = function refresh() { - var req = new XMLHttpRequest(); - req.onload = function() { - if (req.status == 200) { - set_profile(JSON.parse(req.responseText)["profile_json"]); - collectFragmentEventsFromProfile(); - collectUtilizationFromProfile(); - collectFragmentMetricsFromProfile(); - if (last_maxts != maxts) { - renderTimingDiagram(); - last_maxts = maxts; - } - toogleUtilizationVisibility(); - if (profile.child_profiles[0].info_strings.find(({key}) => - key === "End Time").value == "") { - setTimeout(refresh, 1000); +var dbOpenReq = indexedDB.open("imported_queries"); +var db; + +var supported_tabs = ["Query", "Timeline", "Text plan"]; + +function refreshView() { + collectFragmentEventsFromProfile(); + collectUtilizationFromProfile(); + collectFragmentMetricsFromProfile(); + if (last_maxts != maxts) { + renderTimingDiagram(); + last_maxts = maxts; + } + toogleUtilizationVisibility(); +} + + +if (window.location.search.includes("imported")) { + var alertMessage = document.getElementsByClassName("alert alert-danger")[0]; + if (alertMessage) { + alertMessage.remove(); + } + var nav_links = document.getElementsByClassName("nav nav-tabs")[0]; + nav_links = nav_links.getElementsByClassName("nav-link"); + for (var i = 0; i < nav_links.length;) { + if (supported_tabs.includes(nav_links[i].textContent)) { + nav_links[i].href = `${nav_links[i].href}&imported=true`; + i++; + } else { + nav_links[i].parentElement.remove(); + } + } + + dbOpenReq.onsuccess = (e) => { + db = e.target.result; + db.onerror = (e) => { + console.log("IndexedDB error"); + console.log(e); + } + var profileStore = db.transaction("profiles", "readonly").objectStore("profiles"); + var query = {}; + query.id = window.location.search; + query.id = query.id.substring(10, query.id.indexOf("&")); + profileStore.get(query.id).onsuccess = (e) => { + query = e.target.result; + set_profile(query.profile); + refreshView(); + }; + }; +} else { + window.onload = function refreshProfile() { + var req = new XMLHttpRequest(); + req.onload = function() { + if (req.status == 200) { + set_profile(JSON.parse(req.responseText)["profile_json"]); + refreshView(); + if (profile.child_profiles[0].info_strings.find(({key}) => + key === "End Time").value == "") { + setTimeout(refreshProfile, 1000); + } } } + req.open("GET", make_url("/query_timeline?query_id={{query_id}}&json"), true); + req.send(); } - req.open("GET", make_url("/query_timeline?query_id={{query_id}}&json"), true); - req.send(); } window.addEventListener('mousemove', function(e) { diff --git a/www/scripts/query_timeline/fragment_metrics_diagram.js b/www/scripts/query_timeline/fragment_metrics_diagram.js index 54a0a8c68..bb6af59f2 100644 --- a/www/scripts/query_timeline/fragment_metrics_diagram.js +++ b/www/scripts/query_timeline/fragment_metrics_diagram.js @@ -283,6 +283,13 @@ export function updateFragmentMetricsChartOnClick(click_event) { toogleFragmentMetricsVisibility(); } +export function closeFragmentMetricsChart() { + fragment_id_selections.clear(); + fragment_metrics_chart = destroyChart(fragment_metrics_chart, fragment_metrics_diagram); + toogleFragmentMetricsVisibility(); + setTimingDiagramDimensions(); +} + fragment_metrics_resize_bar.addEventListener('mousedown', function dragResizeBarBegin(mousedown_e) { fragment_metrics_resize_bar.removeEventListener('mousedown', dragResizeBarBegin); @@ -294,12 +301,7 @@ fragment_metrics_resize_bar.addEventListener('mousedown', }); }); -fragment_metrics_close_btn.addEventListener('click', function() { - fragment_id_selections.clear(); - fragment_metrics_chart = destroyChart(fragment_metrics_chart, fragment_metrics_diagram); - toogleFragmentMetricsVisibility(); - setTimingDiagramDimensions(); -}); +fragment_metrics_close_btn.addEventListener('click', closeFragmentMetricsChart); fragment_metrics_close_btn.style.height = `${diagram_controls_height}px`; fragment_metrics_close_btn.style.fontSize = `${diagram_controls_height / 2}px`; diff --git a/www/scripts/query_timeline/host_utilization_diagram.js b/www/scripts/query_timeline/host_utilization_diagram.js index 8ef4b69b0..bffd4987e 100644 --- a/www/scripts/query_timeline/host_utilization_diagram.js +++ b/www/scripts/query_timeline/host_utilization_diagram.js @@ -134,6 +134,7 @@ function initializeUtilizationChart() { unload : true }); var chart_width = diagram_width - margin_chart_end - name_width; + prev_utilization_num_samples = 0; host_utilization_resize_bar.style.marginLeft = `${name_width + chart_width / 4}px`; host_utilization_resize_bar.style.width = `${chart_width / 2}px`; host_utilization_resize_bar.style.marginRight = `${chart_width / 4}px`; @@ -302,6 +303,12 @@ export function collectUtilizationFromProfile() { } } +export function destroyUtilizationChart() { + host_utilization_chart = destroyChart(host_utilization_chart, host_utilization_diagram); + toogleUtilizationVisibility(); + setTimingDiagramDimensions(); +} + host_utilization_resize_bar.addEventListener('mousedown', function dragResizeBarBegin(mousedown_e) { host_utilization_resize_bar.removeEventListener('mousedown', dragResizeBarBegin); @@ -316,9 +323,7 @@ host_utilization_resize_bar.addEventListener('mousedown', host_utilization_close_btn.addEventListener('click', function(e) { host_utilization_visible = false; - host_utilization_chart = destroyChart(host_utilization_chart, host_utilization_diagram); - toogleUtilizationVisibility(); - setTimingDiagramDimensions(); + destroyUtilizationChart(); }); host_utilization_close_btn.style.height = `${diagram_controls_height}px`;
