[LNT] Error accessing URL /db_default/v4/nts/profile/ajax/getFunctions; NOT FOUND

[leandron]

Summary

This is a follow-up from https://reviews.llvm.org/D43862, to cover other function the were executing the same code and adding test coverage for the existing functionality.

Original message by Przemek Wirkus:

Hi,
This patch is response to issue with LNT WebUI "Error accessing URL /db_default/v4/nts/profile/ajax/getFunctions; NOT FOUND". This issue appears in certain circumstances when user access Web UI under LNT / nts / profile.

Modified by this patch session query may fetch row(s) with profile_id values set to NULL. In order to fetch only not NULL values this patch modifies way we query DB. Query rows with profile_id set to NULL are not desired when query for LNT/nts/profile data.
Note that session query limits it's search to first element of the query which may have profile_id set to NULL (if entry of this type was imported to LNT database before any entry with profile data).

[leandron]

ping

[jgreenhalgh]

For what it is worth; this is the fix I would have suggested for the issue.

I'm not qualified to review the patch for code style, or project direction; but I would like to see this go in. We need this to display profiles properly in our instance, because our database can get in a state where we return multiple results for the run_id/test_id pair, only some of which have profiles attached.

[MatzeB]

LGTM, though none of us apple people know much about the profile code as it only works on linux right now. But unless someone with more knowledge of the code objects, I'd push this.

[leandron]

Commited as r328614

lnt/server/ui/profile_views.py

 from flask import render_template, current_app
 import os
 import json
 from lnt.server.ui.decorators import v4_route, frontend
 from lnt.server.ui.globals import v4_url_for
 from lnt.server.ui.views import ts_data
 
 
+def _get_sample(session, ts, run_id, test_id):
+    return session.query(ts.Sample) \
+                    .filter(ts.Sample.run_id == run_id) \
+                    .filter(ts.Sample.test_id == test_id) \
+                    .filter(ts.Sample.profile_id.isnot(None)).first()
+
+
 @frontend.route('/profile/admin')
 def profile_admin():
     profileDir = current_app.old_config.profileDir
 
     history_path = os.path.join(profileDir, '_profile-history.json')
     age_path = os.path.join(profileDir, '_profile-age.json')
 
     try:
def v4_profile_ajax_getFunctions():
     ts = request.get_testsuite()
     runid = request.args.get('runid')
     testid = request.args.get('testid')
 
     profileDir = current_app.old_config.profileDir
 
     idx = 0
     tlc = {}
-    sample = session.query(ts.Sample) \
-                    .filter(ts.Sample.run_id == runid) \
-                    .filter(ts.Sample.test_id == testid).first()
+    sample = _get_sample(session, ts, runid, testid)
+
     if sample and sample.profile:
         p = sample.profile.load(profileDir)
         return json.dumps([[n, f] for n, f in p.getFunctions().items()])
     else:
         abort(404)
 
 
 @v4_route("/profile/ajax/getTopLevelCounters")
 def v4_profile_ajax_getTopLevelCounters():
     session = request.session
     ts = request.get_testsuite()
     runids = request.args.get('runids').split(',')
     testid = request.args.get('testid')
 
     profileDir = current_app.old_config.profileDir
 
     idx = 0
     tlc = {}
     for rid in runids:
-        sample = session.query(ts.Sample) \
-                   .filter(ts.Sample.run_id == rid) \
-                   .filter(ts.Sample.test_id == testid).first()
+        sample = _get_sample(session, ts, rid, testid)
         if sample and sample.profile:
             p = sample.profile.load(profileDir)
             for k, v in p.getTopLevelCounters().items():
                 tlc.setdefault(k, [None]*len(runids))[idx] = v
         idx += 1
 
     # If the 1'th counter is None for all keys, truncate the list.
     if all(len(k) > 1 and k[1] is None for k in tlc.values()):
         tlc = {k: [v[0]] for k, v in tlc.items()}
 
     return json.dumps(tlc)
 
 
 @v4_route("/profile/ajax/getCodeForFunction")
 def v4_profile_ajax_getCodeForFunction():
     session = request.session
     ts = request.get_testsuite()
     runid = request.args.get('runid')
     testid = request.args.get('testid')
     f = request.args.get('f')
 
     profileDir = current_app.old_config.profileDir
 
-    sample = session.query(ts.Sample) \
-                    .filter(ts.Sample.run_id == runid) \
-                    .filter(ts.Sample.test_id == testid).first()
+    sample = _get_sample(session, ts, runid, testid)
     if not sample or not sample.profile:
         abort(404)
 
     p = sample.profile.load(profileDir)
     return json.dumps([x for x in p.getCodeForFunction(f)])
 
 
 @v4_route("/profile/<int:testid>/<int:run1_id>")

 def v4_profile(testid, run1_id, run2_id=None):
     session = request.session
     ts = request.get_testsuite()
     profileDir = current_app.old_config.profileDir
 
     try:
         test = session.query(ts.Test).filter(ts.Test.id == testid).one()
         run1 = session.query(ts.Run).filter(ts.Run.id == run1_id).one()
-        sample1 = session.query(ts.Sample) \
-                         .filter(ts.Sample.run_id == run1_id) \
-                         .filter(ts.Sample.test_id == testid).first()
+        sample1 = _get_sample(session, ts, run1_id, testid)
         if run2_id is not None:
             run2 = session.query(ts.Run).filter(ts.Run.id == run2_id).one()
-            sample2 = session.query(ts.Sample) \
-                             .filter(ts.Sample.run_id == run2_id) \
-                             .filter(ts.Sample.test_id == testid).first()
+            sample2 = _get_sample(session, ts, run2_id, testid)
         else:
             run2 = None
             sample2 = None
     except NoResultFound:
         # FIXME: Make this a nicer error page.
         abort(404)
 
     if sample1.profile:

tests/SharedInputs/profile-report.json

+{
+    "Machine": {
+        "Info": {}, 
+        "Name": "e105293.local__clang_DEV__x86_64"
+    }, 
+    "Run": {
+        "End Time": "2016-03-14 14:20:29", 
+        "Info": {
+            "__report_version__": "1", 
+            "cc1_exec_hash": "c431213c8e77728ee825579745d700e79c3521f2", 
+            "cc_alt_src_branch": "llvm/trunk", 
+            "cc_alt_src_revision": "154329", 
+            "cc_as_version": "unrecognized argument vector: ('-c', '-Wa,-v', '-o', '/dev/null', '-x', 'assembler', '/dev/null')", 
+            "cc_build": "DEV", 
+            "cc_dumpmachine": "x86_64-apple-darwin11.0.0", 
+            "cc_exec_hash": "c431213c8e77728ee825579745d700e79c3521f2", 
+            "cc_ld_version": "unrecognized argument vector: ('-Wl,-v', '-o', '/dev/null', '/tmp/tmpzWa737.c')", 
+            "cc_name": "clang", 
+            "cc_src_branch": "trunk", 
+            "cc_src_revision": "154331", 
+            "cc_target": "x86_64-apple-darwin11.0.0", 
+            "cc_target_assembly": "; ModuleID = '/dev/null'\ntarget datalayout = \"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64\"\ntarget triple = \"x86_64-apple-darwin11.0.0\"", 
+            "cc_version": "clang version 3.1 (trunk 154331) (llvm/trunk 154329)\nTarget: x86_64-apple-darwin11.3.0\nThread model: posix\nInstalledDir: /home/foo/bin\n\n \"/Users/jammol01/Code/lnt/tests/SharedInputs/FakeCompilers/clang-r154331\" \"-cc1\" \"-E\" ... more boring stuff here ...", 
+            "cc_version_number": "3.1", 
+            "inferred_run_order": "154331", 
+            "run_order": "154331", 
+            "tag": "nts"
+        }, 
+        "Start Time": "2016-03-14 14:20:28"
+    }, 
+    "Tests": [
+        {
+            "Data": [
+                1.4
+            ], 
+            "Info": {}, 
+            "Name": "nts.foo.exec"
+        }, 
+        {
+            "Data": [
+                1.3
+            ], 
+            "Info": {}, 
+            "Name": "nts.foo.compile"
+        }, 
+        {
+            "Data": [
+                1.5
+            ], 
+            "Info": {}, 
+            "Name": "nts.foo.score"
+        }, 
+        {
+            "Data": [
+                "xyz"
+            ], 
+            "Info": {}, 
+            "Name": "nts.foo.hash"
+        }, 
+        {
+            "Data": [
+                "eJxNj8EOgjAMhu99Cm9wULMOEHgBE888QdkASWCQFWJ8e1v04JIt+9f//7qmfkVoEj8yMXdzO70v/RJn2hJYrRQiveSWATdJvwe3jUtgecgh9Wsh9T6gyJvKUjm0kegK0mmt9UCjJUSgB5q8KsobUJOQ96dozr8tAbRApPbssOeCcm83ddoLC7ijMcA/RGUUwXt7iviPEDLJN92yh62LR7I8aBUMysgLnaKNFNzzMo8y7uGplQ4sa/j6rfn60WYaGdRhtT9fP5+JUW4="
+            ], 
+            "Info": {}, 
+            "Name": "nts.foo.profile"
+        }
+    ]
+}

tests/server/db/ImportProfile.py

 # Check that we can import profiles into a test DB
 #
 # We first construct a temporary LNT instance.
 # RUN: rm -rf %t.install
 # RUN: lnt create %t.install
 
 # Import the test set
-# RUN: lnt import %t.install %S/Inputs/profile-report.json \
+# RUN: lnt import %t.install  %{shared_inputs}/profile-report.json \
 # RUN:   --show-sample-count > %t2.log
 # RUN: ls %t.install/data/profiles
 # RUN: python %s %t.install
 
 import os, sys, glob
 from lnt.testing.profile.profilev1impl import ProfileV1
 
 profile = glob.glob('%s/data/profiles/*.lntprof' % sys.argv[1])[0]
 assert ProfileV1.checkFile(profile)

tests/server/db/Inputs/profile-report.json

-{
-    "Machine": {
-        "Info": {}, 
-        "Name": "e105293.local__clang_DEV__x86_64"
-    }, 
-    "Run": {
-        "End Time": "2016-03-14 14:20:29", 
-        "Info": {
-            "__report_version__": "1", 
-            "cc1_exec_hash": "c431213c8e77728ee825579745d700e79c3521f2", 
-            "cc_alt_src_branch": "llvm/trunk", 
-            "cc_alt_src_revision": "154329", 
-            "cc_as_version": "unrecognized argument vector: ('-c', '-Wa,-v', '-o', '/dev/null', '-x', 'assembler', '/dev/null')", 
-            "cc_build": "DEV", 
-            "cc_dumpmachine": "x86_64-apple-darwin11.0.0", 
-            "cc_exec_hash": "c431213c8e77728ee825579745d700e79c3521f2", 
-            "cc_ld_version": "unrecognized argument vector: ('-Wl,-v', '-o', '/dev/null', '/tmp/tmpzWa737.c')", 
-            "cc_name": "clang", 
-            "cc_src_branch": "trunk", 
-            "cc_src_revision": "154331", 
-            "cc_target": "x86_64-apple-darwin11.0.0", 
-            "cc_target_assembly": "; ModuleID = '/dev/null'\ntarget datalayout = \"e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64\"\ntarget triple = \"x86_64-apple-darwin11.0.0\"", 
-            "cc_version": "clang version 3.1 (trunk 154331) (llvm/trunk 154329)\nTarget: x86_64-apple-darwin11.3.0\nThread model: posix\nInstalledDir: /home/foo/bin\n\n \"/Users/jammol01/Code/lnt/tests/SharedInputs/FakeCompilers/clang-r154331\" \"-cc1\" \"-E\" ... more boring stuff here ...", 
-            "cc_version_number": "3.1", 
-            "inferred_run_order": "154331", 
-            "run_order": "154331", 
-            "tag": "nts"
-        }, 
-        "Start Time": "2016-03-14 14:20:28"
-    }, 
-    "Tests": [
-        {
-            "Data": [
-                1.4
-            ], 
-            "Info": {}, 
-            "Name": "nts.foo.exec"
-        }, 
-        {
-            "Data": [
-                1.3
-            ], 
-            "Info": {}, 
-            "Name": "nts.foo.compile"
-        }, 
-        {
-            "Data": [
-                1.5
-            ], 
-            "Info": {}, 
-            "Name": "nts.foo.score"
-        }, 
-        {
-            "Data": [
-                "xyz"
-            ], 
-            "Info": {}, 
-            "Name": "nts.foo.hash"
-        }, 
-        {
-            "Data": [
-                "eJxNj8EOgjAMhu99Cm9wULMOEHgBE888QdkASWCQFWJ8e1v04JIt+9f//7qmfkVoEj8yMXdzO70v/RJn2hJYrRQiveSWATdJvwe3jUtgecgh9Wsh9T6gyJvKUjm0kegK0mmt9UCjJUSgB5q8KsobUJOQ96dozr8tAbRApPbssOeCcm83ddoLC7ijMcA/RGUUwXt7iviPEDLJN92yh62LR7I8aBUMysgLnaKNFNzzMo8y7uGplQ4sa/j6rfn60WYaGdRhtT9fP5+JUW4="
-            ], 
-            "Info": {}, 
-            "Name": "nts.foo.profile"
-        }
-    ]
-}

tests/server/ui/V4Pages.py

 # Perform basic sanity checking of the V4 UI pages.
 #
 # create temporary instance
 # Cleanup temporary directory in case one remained from a previous run - also
 # see PR9904.
 # RUN: rm -rf %t.instance
 # RUN: python %{shared_inputs}/create_temp_instance.py \
 # RUN:   %s %{shared_inputs}/SmallInstance %t.instance \
 # RUN:   %S/Inputs/V4Pages_extra_records.sql
+
+# Import a profile
+# RUN: lnt import %t.instance %{shared_inputs}/profile-report.json
 #
 # RUN: python %s %t.instance %{tidylib}
 
 import logging
 import re
 import sys
 import xml.etree.ElementTree as ET
 from htmlentitydefs import name2codepoint
def main():
     check_code(client, '/v4/compile/machine/-1', expected_code=HTTP_NOT_FOUND)
     check_code(client, '/v4/compile/machine/a', expected_code=HTTP_NOT_FOUND)
 
     # Check the compare machine form gives correct redirects.
     resp = check_code(client, '/v4/nts/machine/2/compare?compare_to_id=3', expected_code=HTTP_REDIRECT)
     assert resp.headers['Location'] == "http://localhost/db_default/v4/nts/9?compare_to=4"
     resp = check_code(client, '/v4/nts/machine/3/compare?compare_to_id=2', expected_code=HTTP_REDIRECT)
     assert resp.headers['Location'] == "http://localhost/db_default/v4/nts/4?compare_to=9"
-    
     # Get the order summary page.
     check_html(client, '/v4/compile/all_orders')
 
     # Get an order page.
     check_html(client, '/v4/compile/order/3')
 
     # Get a run result page (and associated views).
     check_html(client, '/v4/compile/1')
def main():
     check_html(client, '/v4/nts/regressions/')
     check_html(client, '/v4/nts/regressions/?machine_filter=machine2')
     check_html(client, '/v4/nts/regressions/?machine_filter=machine0')
 
     check_html(client, '/v4/nts/regressions/1')
 
     check_json(client, '/v4/nts/regressions/1?json=True')
 
+    # Check 404 is issues for inexistent Code
+    check_code(client, 'v4/nts/profile/9999/9999', expected_code=HTTP_NOT_FOUND)
+
+    # Profile Viewer Ajax functions
+    # Check profiles page is responsive with expected IDs
+    check_code(client, 'v4/nts/profile/10/10')
+    # Check ajax call
+    functions = check_json(client, 'v4/nts/profile/ajax/getFunctions?runid=10&testid=10')
+    number_of_functions = len(functions)
+    first_function_name = functions[0][0]
+    assert 1 == number_of_functions
+    assert "fn1" == first_function_name
+
+    top_level_counters = check_json(client, 'v4/nts/profile/ajax/getTopLevelCounters?runids=10&testid=10')
+    assert "cycles" in top_level_counters
+    assert "branch-misses" in top_level_counters
+
+    code_for_fn = check_json(client, 'v4/nts/profile/ajax/getCodeForFunction?runid=10&testid=10&f=fn1')
+    lines_in_function = len(code_for_fn)
+    assert 2 == lines_in_function
+
     # Make sure the new option does not break anything
     check_html(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&submit=Update')
     check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true&submit=Update')
     check_html(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2')
     check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true')
     app.testing = False
     error_page = check_html(client, '/explode', expected_code=500)
     assert "integer division or modulo by zero" in error_page.data