From 1652a7529b979077689369efeb8af5ab3b5bac30 Mon Sep 17 00:00:00 2001 From: Dan Callaghan Date: Sep 25 2017 07:28:40 +0000 Subject: /results/latest: use a single query with subquery join ... instead of issuing a separate query for every possible matching test case, of which there could be a very large number. This improves the performance of the /results/latest endpoint, which is currently domainated by roundtrip time to the database for all the queries. Fixes #84. --- diff --git a/resultsdb/controllers/api_v2.py b/resultsdb/controllers/api_v2.py index 520c3d6..0bcde79 100644 --- a/resultsdb/controllers/api_v2.py +++ b/resultsdb/controllers/api_v2.py @@ -316,13 +316,14 @@ def select_results(since_start=None, since_end=None, outcomes=None, groups=None, # Checks if the sort parameter specified in the request is valid before querying. # Sorts by submit_time in a descending order if the sort parameter is absent or invalid. query_sorted = False - sort_match = re.match(r'^(?Pasc|desc):(?P.+)$', _sort) - if sort_match: - if sort_match.group('column') == 'submit_time': - sort_order = {'asc': db.asc, 'desc': db.desc}[sort_match.group('order')] - sort_column = getattr(Result, sort_match.group('column')) - q = db.session.query(Result).order_by(sort_order(sort_column)) - query_sorted = True + if _sort: + sort_match = re.match(r'^(?Pasc|desc):(?P.+)$', _sort) + if sort_match: + if sort_match.group('column') == 'submit_time': + sort_order = {'asc': db.asc, 'desc': db.desc}[sort_match.group('order')] + sort_column = getattr(Result, sort_match.group('column')) + q = db.session.query(Result).order_by(sort_order(sort_column)) + query_sorted = True if not query_sorted: q = db.session.query(Result).order_by(db.desc(Result.submit_time)) @@ -447,22 +448,35 @@ def get_results_latest(): return p['error'] args = p['args'] - q = select_testcases(','.join(args['testcases']), ','.join(args['testcases:like'])) - testcases = q.all() - - results = [] - for testcase in testcases: - q = select_results( - since_start=args['since']['start'], - since_end=args['since']['end'], - groups=args['groups'], - testcases=[testcase.name], - result_data=p['result_data'], - _sort=args['_sort'], - ) - result = q.first() - if result: - results.append(result) + + q = select_results( + since_start=args['since']['start'], + since_end=args['since']['end'], + groups=args['groups'], + testcases=args['testcases'], + testcases_like=args['testcases:like'], + result_data=p['result_data'], + _sort=args['_sort'], + ) + + # Produce a subquery with the same filter criteria as above *except* + # test case name, which we group by and join on. + sq = select_results( + since_start=args['since']['start'], + since_end=args['since']['end'], + groups=args['groups'], + result_data=p['result_data'], + )\ + .order_by(None)\ + .with_entities( + Result.testcase_name.label('testcase_name'), + db.func.max(Result.submit_time).label('max_submit_time'))\ + .group_by(Result.testcase_name)\ + .subquery() + q = q.join(sq, db.and_(Result.testcase_name == sq.c.testcase_name, + Result.submit_time == sq.c.max_submit_time)) + + results = q.all() return jsonify(dict( data=[SERIALIZE(o) for o in results], diff --git a/testing/functest_api_v20.py b/testing/functest_api_v20.py index f17c83d..605771f 100644 --- a/testing/functest_api_v20.py +++ b/testing/functest_api_v20.py @@ -788,9 +788,9 @@ class TestFuncApiV20(): data = json.loads(r.data) assert len(data['data']) == 2 - assert data['data'][0]['testcase']['name'] == self.ref_testcase_name - assert data['data'][0]['outcome'] == "FAILED" - assert data['data'][1]['testcase']['name'] == self.ref_testcase_name + ".1" + assert data['data'][0]['testcase']['name'] == self.ref_testcase_name + ".1" + assert data['data'][1]['testcase']['name'] == self.ref_testcase_name + assert data['data'][1]['outcome'] == "FAILED" def test_get_results_latest_modifiers(self): self.helper_create_testcase() @@ -818,28 +818,28 @@ class TestFuncApiV20(): data = json.loads(r.data) assert len(data['data']) == 2 - assert data['data'][0]['testcase']['name'] == self.ref_testcase_name + assert data['data'][0]['testcase']['name'] == self.ref_testcase_name + ".1" assert data['data'][0]['outcome'] == "FAILED" - assert data['data'][1]['testcase']['name'] == self.ref_testcase_name + ".1" + assert data['data'][1]['testcase']['name'] == self.ref_testcase_name assert data['data'][1]['outcome'] == "FAILED" r = self.app.get('/api/v2.0/results/latest?testcases:like=*') data = json.loads(r.data) assert len(data['data']) == 2 - assert data['data'][0]['testcase']['name'] == self.ref_testcase_name + assert data['data'][0]['testcase']['name'] == self.ref_testcase_name + ".1" assert data['data'][0]['outcome'] == "FAILED" - assert data['data'][1]['testcase']['name'] == self.ref_testcase_name + ".1" + assert data['data'][1]['testcase']['name'] == self.ref_testcase_name assert data['data'][1]['outcome'] == "FAILED" r = self.app.get('/api/v2.0/results/latest?groups=%s' % self.ref_group_uuid) data = json.loads(r.data) assert len(data['data']) == 2 - assert data['data'][0]['testcase']['name'] == self.ref_testcase_name - assert data['data'][0]['outcome'] == "FAILED" - assert data['data'][1]['testcase']['name'] == self.ref_testcase_name + ".1" - assert data['data'][1]['outcome'] == "PASSED" + assert data['data'][0]['testcase']['name'] == self.ref_testcase_name + ".1" + assert data['data'][0]['outcome'] == "PASSED" + assert data['data'][1]['testcase']['name'] == self.ref_testcase_name + assert data['data'][1]['outcome'] == "FAILED" def test_message_publication(self): self.helper_create_result()