[LNT] Python 3 support: Parse HTML as text

[thopre]

Summary

Several tests and the testsuite run import code parse text read from the
data attribute of a Flask response object (werkzeug.wrappers.Response).
However, in Python3 reading from that attribute yields binary data.
The attribute is in fact a property whose getter get_data() as a as_text
parameter that allows to request the result to be a string instead.

This commit adapts all reads from that property to request strings
instead of binary data (the default), except for a few cases in
tests/server/ui/V4Pages.py where the data is fed to ElementTree's
fromstring method in get_xml_tree() which tries to reencode it in ascii
in Python 2 mode if it gets a unicode string.

[thopre]

Merge more similar cases into this patch

[PrzemekWirkus]

LGTM but someone else must approve.

tests/server/ui/test_api_modify.py

68

Nitpick:

We indeed tend toward assertEqual for Python 3.
You've already replaced one assertEquala with assertEqual in this patch, could you clean this one up as well?

[thopre]

tests/server/ui/test_api_modify.py

68

I've made a separate diff fixing all existing use of assertEquals: https://reviews.llvm.org/D71057

[thopre]

Committed as b2e114bd7f7c2d6fe08edbe0c5811768e703727d

lnt/server/ui/api.py

class Runs(Resource):
     method_decorators = [in_db]
 
     @staticmethod
     @requires_auth_token
     def post():
         """Add a new run into the lnt database"""
         session = request.session
         db = request.get_db()
-        data = request.data
+        data = request.get_data(as_text=True)
         select_machine = request.values.get('select_machine', 'match')
         merge = request.values.get('merge', None)
         result = lnt.util.ImportData.import_from_string(
             current_app.old_config, g.db_name, db, session, g.testsuite_name,
             data, select_machine=select_machine, merge_run=merge)
 
         error = result['error']
         if error is not None:

tests/server/db/Migrations.py

def sanity_check_instance(instance_path):
     # Create a test client.
     client = app.test_client()
 
     # Fetch the index page.
     index = client.get('/')
 
     # Visit all the test suites.
     test_suite_link_rex = re.compile("""  <a href="(.*)">(.*)</a><br>""")
-    test_suite_list_start = index.data.index("<h3>Test Suites</h3>")
-    test_suite_list_end = index.data.index("</div>", test_suite_list_start)
-    for ln in index.data[test_suite_list_start:test_suite_list_end].split("\n"):
+    data = index.get_data(as_text=True)
+    test_suite_list_start = data.index("<h3>Test Suites</h3>")
+    test_suite_list_end = data.index("</div>", test_suite_list_start)
+    for ln in data[test_suite_list_start:test_suite_list_end].split("\n"):
         # Ignore non-matching lines.
         print(ln, file=sys.stderr)
         m = test_suite_link_rex.match(ln)
         if not m:
             continue
 
         # We found a test suite link...
         link,name = m.groups()

tests/server/ui/V4Pages.py

def check_code(client, url, expected_code=HTTP_OK, data_to_send=None):
     assert resp.status_code == expected_code, \
         "Call to %s returned: %d, not the expected %d" % (url, resp.status_code,
                                                           expected_code)
     return resp
 
 
 def check_html(client, url, expected_code=HTTP_OK, data_to_send=None):
     resp = check_code(client, url, expected_code, data_to_send)
-    validate_html(resp.data)
+    validate_html(resp.get_data(as_text=True))
     return resp
 
 
 def check_json(client, url, expected_code=HTTP_OK, data_to_send=None):
     """Call a flask url, make sure the return code is good,
     and grab reply data from the json payload."""
     return json.loads(check_code(client, url, expected_code,
                       data_to_send=data_to_send).data)
for parent in table_parent_elements:
                     return child
             elif child.tag.startswith('h') and child.text == table_heading:
                 found_header = True
     return None
 
 
 def check_nr_machines_reported(client, url, expected_nr_machines):
     resp = check_code(client, url)
-    html = resp.data
+    html = resp.get_data(as_text=sys.version_info[0] >= 3)
     tree = get_xml_tree(html)
     # look for the table containing the machines on the page.
     # do this by looking for the title containing "Reported Machine Order"
     # and assuming that the first <table> at the same level after it is the
     # one we're looking for.
     reported_machine_order_table = \
         find_table_with_heading(tree, 'Reported Machine Order')
     if reported_machine_order_table is None:
         nr_machines = 0
     else:
         nr_machines = len(reported_machine_order_table.findall("./tbody/tr"))
     assert expected_nr_machines == nr_machines
 
 
 def convert_html_to_text(element):
     return ("".join(element.itertext()))
 
 
 def get_table_by_header(client, url, table_header):
     resp = check_code(client, url)
-    html = resp.data
+    html = resp.get_data(as_text=sys.version_info[0] >= 3)
     tree = get_xml_tree(html)
     table = find_table_with_heading(tree, table_header)
     assert table is not None, \
         "Couldn't find table with header '%s'" % table_header
     return table
 
 
 def get_results_table(client, url, fieldname):
def check_body_nr_tests_table(client, url, expected_content):
     table_header = "Number of Tests Seen"
     table = get_table_by_header(client, url, table_header)
     check_table_content(table, expected_content)
 
 
 def check_producer_label(client, url, label):
     table_header = "Produced by"
     resp = check_code(client, url)
-    tree = get_xml_tree(resp.data)
+    tree = get_xml_tree(resp.get_data(as_text=sys.version_info[0] >= 3))
     table = find_table_by_thead_content(tree, table_header)
     check_row_is_in_table(table, label)
 
 
 def get_sparkline(table, testname, machinename):
     body_content = [[cell
                      for cell in row.findall("./td")]
                     for row in table.findall("./tbody/tr")]
def main():
     assert r.status_code == HTTP_REDIRECT
 
     # Try with redirect.
     r = client.post('/v4/nts/order/3',
                     data=form_data,
                     follow_redirects=True)
     assert r.status_code == HTTP_OK
     # Should see baseline displayed in page body.
-    assert "Baseline - foo_baseline" in r.data
+    assert "Baseline - foo_baseline" in r.get_data(as_text=True)
 
     # Now demote it.
     data2 = dict(name="foo_baseline",
                  description="foo_description",
                  update=False,
                  promote=False,
                  demote=True)
     r = client.post('/v4/nts/order/3', data=data2, follow_redirects=True)
     assert r.status_code == HTTP_OK
     # Baseline should no longer be shown in page baseline.
-    assert "Baseline - foo_baseline" not in r.data
+    assert "Baseline - foo_baseline" not in r.get_data(as_text=True)
 
     # Leave a baseline in place for the rest of the tests.
     client.post('/v4/nts/order/3', data=form_data)
 
     check_code(client, '/v4/nts/set_baseline/1', expected_code=HTTP_REDIRECT)
     with app.test_client() as c:
         c.get('/v4/nts/set_baseline/1')
         session.get('baseline-default-nts') == 1
def main():
 
     # Make sure the new option does not break anything
     check_html(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&submit=Update')
     check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true&submit=Update')
     check_html(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2')
     check_json(client, '/db_default/v4/nts/graph?switch_min_mean=yes&plot.0=1.3.2&json=true')
     app.testing = False
     error_page = check_html(client, '/explode', expected_code=500)
-    assert "integer division or modulo by zero" in error_page.data
+    assert "integer division or modulo by zero" in error_page.get_data(as_text=True)
 
     error_page = check_html(client, '/gone', expected_code=404)
-    assert "test" in error_page.data
+    assert "test" in error_page.get_data(as_text=True)
 
     check_html(client, '/sleep?timeout=0', expected_code=200)
 
     check_html(client, '/db_default/summary_report')
 
     check_html(client, '/rules')
     check_html(client, '/log')
     resp = check_code(client, '/__health')
-    assert resp.data == "Ok"
+    assert resp.get_data(as_text=True) == "Ok"
     resp = check_code(client, '/ping')
-    assert resp.data == "pong"
+    assert resp.get_data(as_text=True) == "pong"
 
     # Check we can convert a sample into a graph page.
     graph_to_sample = check_code(client, '/db_default/v4/nts/graph_for_sample/10/compile_time?foo=bar',
                                  expected_code=HTTP_REDIRECT)
     assert graph_to_sample.headers['Location'] == "http://localhost/db_default/v4/nts/graph?foo=bar&plot.0=2.6.0"
 
     # Check that is we ask for a sample or invalid field, we explode with 400s.
     check_code(client, '/db_default/v4/nts/graph_for_sample/10000/compile_time?foo=bar',

tests/server/ui/test_api_modify.py

def test_00_update_machine(self):
         json_data = json.dumps(data)
         resp = client.put('api/db_default/v4/nts/machines/1', data=json_data,
                           headers={'AuthToken': 'test_token'})
         self.assertEqual(resp.status_code, 200)
 
         machine_after = check_json(client, 'api/db_default/v4/nts/machines/1')
         machine_after = machine_after['machine']
         for key in ('hardware', 'os', 'hostname', 'new_parameter', 'uname'):
             self.assertEquals(machine_after.get(key, None),
                               data['machine'].get(key, None))
 
     def test_00_rename_machine(self):
         """Check rename POST request to /machines/n"""
         client = self.client
 
         # Make sure the environment is as expected.
         j = check_json(client, 'api/db_default/v4/nts/machines/1')
def test_02_delete_machine(self):
 
         resp = client.delete('api/db_default/v4/nts/machines/2',
                              headers={'AuthToken': 'wrong token'})
         self.assertEqual(resp.status_code, 401)
 
         resp = client.delete('api/db_default/v4/nts/machines/2',
                              headers={'AuthToken': 'test_token'})
         self.assertEqual(resp.status_code, 200)
-        self.assertEqual(resp.get_data(),
+        self.assertEqual(resp.get_data(as_text=True),
                          '''Deleting runs 3 5 6 7 8 9 (6/6)
 Deleted machine machine2:2
 ''')
 
         resp = client.get('api/db_default/v4/nts/machines/2')
         self.assertEqual(resp.status_code, 404)
 
         for run_id in run_ids:

tests/server/ui/test_matrix_page.py

def setUp(self):
         self.client = app.test_client()
 
     def test_config_errors(self):
         """Does passing bad arguments to matrix view error correctly.
         """
         client = self.client
         reply = check_code(client, '/v4/nts/matrix',
                            expected_code=HTTP_NOT_FOUND)
-        self.assertIn("Request requires some data arguments.", reply.data)
+        self.assertIn("Request requires some data arguments.",
+                      reply.get_data(as_text=True))
 
         reply = check_code(client, '/v4/nts/matrix?plot.0=1.1.1',
                            expected_code=HTTP_NOT_FOUND)
-        self.assertIn("No data found.", reply.data)
+        self.assertIn("No data found.", reply.get_data(as_text=True))
 
         reply = check_code(client, '/v4/nts/matrix?plot.0=a.2.0',
                            expected_code=HTTP_BAD_REQUEST)
-        self.assertIn("malformed", reply.data)
+        self.assertIn("malformed", reply.get_data(as_text=True))
 
         reply = check_code(client, '/v4/nts/matrix?plot.0=999.0.0',
                            expected_code=HTTP_NOT_FOUND)
-        self.assertIn("Invalid machine", reply.data)
+        self.assertIn("Invalid machine", reply.get_data(as_text=True))
         reply = check_code(client, '/v4/nts/matrix?plot.0=1.999.0',
                            expected_code=HTTP_NOT_FOUND)
-        self.assertIn("Invalid test", reply.data)
+        self.assertIn("Invalid test", reply.get_data(as_text=True))
         reply = check_code(client, '/v4/nts/matrix?plot.0=1.1.999',
                            expected_code=HTTP_NOT_FOUND)
-        self.assertIn("Invalid field", reply.data)
+        self.assertIn("Invalid field", reply.get_data(as_text=True))
 
     def test_matrix_view(self):
         """Does the page load with the data as expected.
         """
         client = self.client
         reply = check_html(client, '/v4/nts/matrix?plot.0=2.6.2')
         # Set a baseline and run again.
         form_data = dict(name="foo_baseline",
                          description="foo_description",
                          prmote=True)
         rc = client.post('/v4/nts/order/6', data=form_data)
-        self.assertEquals(rc.status_code, HTTP_REDIRECT)
+        self.assertEqual(rc.status_code, HTTP_REDIRECT)
         check_code(client, '/v4/nts/set_baseline/1',
                    expected_code=HTTP_REDIRECT)
 
         reply = check_html(client, '/v4/nts/matrix?plot.0=2.6.2')
         # Make sure the data is in the page.
-        self.assertIn("test6", reply.data)
-        self.assertIn("1.0000", reply.data)
-        self.assertIn("1.2000", reply.data)
+        data = reply.get_data(as_text=True)
+        self.assertIn("test6", data)
+        self.assertIn("1.0000", data)
+        self.assertIn("1.2000", data)
 
         reply = check_html(client, '/v4/nts/matrix?plot.0=2.6.2&limit=1')
 
 
 if __name__ == '__main__':
     unittest.main(argv=[sys.argv[0], ])