infra/cifuzz/get_coverage_test.py - platform/external/oss-fuzz - Git at Google

 # Copyright 2021 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #      http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Tests for get_coverage.py"""
 import os
 import json
 import unittest
 from unittest import mock

 from pyfakefs import fake_filesystem_unittest
 import pytest

 import get_coverage

 # pylint: disable=protected-access

 TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                               'test_data')

 PROJECT_NAME = 'curl'
 REPO_PATH = '/src/curl'
 FUZZ_TARGET = 'curl_fuzzer'
 PROJECT_COV_JSON_FILENAME = 'example_curl_cov.json'
 FUZZ_TARGET_COV_JSON_FILENAME = 'example_curl_fuzzer_cov.json'
 INVALID_TARGET = 'not-a-fuzz-target'

 with open(os.path.join(TEST_DATA_PATH,
                        PROJECT_COV_JSON_FILENAME),) as cov_file_handle:
   PROJECT_COV_INFO = json.loads(cov_file_handle.read())


 class GetOssFuzzFuzzerStatsDirUrlTest(unittest.TestCase):
   """Tests _get_oss_fuzz_fuzzer_stats_dir_url."""

   @mock.patch('http_utils.get_json_from_url',
               return_value={
                   'fuzzer_stats_dir':
                       'gs://oss-fuzz-coverage/systemd/fuzzer_stats/20210303'
               })
   def test_get_valid_project(self, mock_get_json_from_url):
     """Tests that a project's coverage report can be downloaded and parsed.

     NOTE: This test relies on the PROJECT_NAME repo's coverage report.
     The "example" project was not used because it has no coverage reports.
     """
     result = get_coverage._get_oss_fuzz_fuzzer_stats_dir_url(PROJECT_NAME)
     (url,), _ = mock_get_json_from_url.call_args
     self.assertEqual(
         'https://storage.googleapis.com/oss-fuzz-coverage/'
         'latest_report_info/curl.json', url)

     expected_result = (
         'https://storage.googleapis.com/oss-fuzz-coverage/systemd/fuzzer_stats/'
         '20210303')
     self.assertEqual(result, expected_result)

   def test_get_invalid_project(self):
     """Tests that passing a bad project returns None."""
     self.assertIsNone(
         get_coverage._get_oss_fuzz_fuzzer_stats_dir_url('not-a-proj'))


 class OSSFuzzCoverageGetTargetCoverageTest(unittest.TestCase):
   """Tests OSSFuzzCoverage.get_target_coverage."""

   def setUp(self):
     with mock.patch('get_coverage._get_oss_fuzz_latest_cov_report_info',
                     return_value=PROJECT_COV_INFO):
       self.oss_fuzz_coverage = get_coverage.OSSFuzzCoverage(
           REPO_PATH, PROJECT_NAME)

   @mock.patch('http_utils.get_json_from_url', return_value={})
   def test_valid_target(self, mock_get_json_from_url):
     """Tests that a target's coverage report can be downloaded and parsed."""
     self.oss_fuzz_coverage.get_target_coverage(FUZZ_TARGET)
     (url,), _ = mock_get_json_from_url.call_args
     self.assertEqual(
         'https://storage.googleapis.com/oss-fuzz-coverage/'
         'curl/fuzzer_stats/20200226/curl_fuzzer.json', url)

   def test_invalid_target(self):
     """Tests that passing an invalid target coverage report returns None."""
     self.assertIsNone(
         self.oss_fuzz_coverage.get_target_coverage(INVALID_TARGET))

   @mock.patch('get_coverage._get_oss_fuzz_latest_cov_report_info',
               return_value=None)
   def test_invalid_project_json(self, _):  # pylint: disable=no-self-use
     """Tests an invalid project JSON results in None being returned."""
     with pytest.raises(get_coverage.CoverageError):
       get_coverage.OSSFuzzCoverage(REPO_PATH, PROJECT_NAME)


 def _get_expected_curl_covered_file_list():
   """Returns the expected covered file list for
   FUZZ_TARGET_COV_JSON_FILENAME."""
   curl_files_list_path = os.path.join(TEST_DATA_PATH,
                                       'example_curl_file_list.json')
   with open(curl_files_list_path) as file_handle:
     return json.loads(file_handle.read())


 def _get_example_curl_coverage():
   """Returns the contents of the fuzzer stats JSON file for
   FUZZ_TARGET_COV_JSON_FILENAME."""
   with open(os.path.join(TEST_DATA_PATH,
                          FUZZ_TARGET_COV_JSON_FILENAME)) as file_handle:
     return json.loads(file_handle.read())


 class OSSFuzzCoverageGetFilesCoveredByTargetTest(unittest.TestCase):
   """Tests OSSFuzzCoverage.get_files_covered_by_target."""

   def setUp(self):
     with mock.patch('get_coverage._get_oss_fuzz_latest_cov_report_info',
                     return_value=PROJECT_COV_INFO):
       self.oss_fuzz_coverage = get_coverage.OSSFuzzCoverage(
           REPO_PATH, PROJECT_NAME)

   def test_valid_target(self):
     """Tests that covered files can be retrieved from a coverage report."""
     fuzzer_cov_data = _get_example_curl_coverage()
     with mock.patch('get_coverage.OSSFuzzCoverage.get_target_coverage',
                     return_value=fuzzer_cov_data):
       file_list = self.oss_fuzz_coverage.get_files_covered_by_target(
           FUZZ_TARGET)

     expected_file_list = _get_expected_curl_covered_file_list()
     self.assertCountEqual(file_list, expected_file_list)

   def test_invalid_target(self):
     """Tests passing invalid fuzz target returns None."""
     self.assertIsNone(
         self.oss_fuzz_coverage.get_files_covered_by_target(INVALID_TARGET))


 class FilesystemCoverageGetFilesCoveredByTargetTest(
     fake_filesystem_unittest.TestCase):
   """Tests FilesystemCoverage.get_files_covered_by_target."""

   def setUp(self):
     _fuzzer_cov_data = _get_example_curl_coverage()
     self._expected_file_list = _get_expected_curl_covered_file_list()
     self.coverage_path = '/coverage'
     self.filesystem_coverage = get_coverage.FilesystemCoverage(
         REPO_PATH, self.coverage_path)
     self.setUpPyfakefs()
     self.fs.create_file(os.path.join(self.coverage_path, 'fuzzer_stats',
                                      FUZZ_TARGET + '.json'),
                         contents=json.dumps(_fuzzer_cov_data))

   def test_valid_target(self):
     """Tests that covered files can be retrieved from a coverage report."""
     file_list = self.filesystem_coverage.get_files_covered_by_target(
         FUZZ_TARGET)
     self.assertCountEqual(file_list, self._expected_file_list)

   def test_invalid_target(self):
     """Tests passing invalid fuzz target returns None."""
     self.assertIsNone(
         self.filesystem_coverage.get_files_covered_by_target(INVALID_TARGET))


 class IsFileCoveredTest(unittest.TestCase):
   """Tests for is_file_covered."""

   def test_is_file_covered_covered(self):
     """Tests that is_file_covered returns True for a covered file."""
     file_coverage = {
         'filename': '/src/systemd/src/basic/locale-util.c',
         'summary': {
             'regions': {
                 'count': 204,
                 'covered': 200,
                 'notcovered': 200,
                 'percent': 98.03
             }
         }
     }
     self.assertTrue(get_coverage.is_file_covered(file_coverage))

   def test_is_file_covered_not_covered(self):
     """Tests that is_file_covered returns False for a not covered file."""
     file_coverage = {
         'filename': '/src/systemd/src/basic/locale-util.c',
         'summary': {
             'regions': {
                 'count': 204,
                 'covered': 0,
                 'notcovered': 0,
                 'percent': 0
             }
         }
     }
     self.assertFalse(get_coverage.is_file_covered(file_coverage))


 class GetOssFuzzLatestCovReportInfo(unittest.TestCase):
   """Tests that _get_oss_fuzz_latest_cov_report_info works as
   intended."""

   PROJECT = 'project'
   LATEST_REPORT_INFO_URL = ('https://storage.googleapis.com/oss-fuzz-coverage/'
                             'latest_report_info/project.json')

   @mock.patch('logging.error')
   @mock.patch('http_utils.get_json_from_url', return_value={'coverage': 1})
   def test_get_oss_fuzz_latest_cov_report_info(self, mock_get_json_from_url,
                                                mock_error):
     """Tests that _get_oss_fuzz_latest_cov_report_info works as intended."""
     result = get_coverage._get_oss_fuzz_latest_cov_report_info(self.PROJECT)
     self.assertEqual(result, {'coverage': 1})
     mock_error.assert_not_called()
     mock_get_json_from_url.assert_called_with(self.LATEST_REPORT_INFO_URL)

   @mock.patch('logging.error')
   @mock.patch('http_utils.get_json_from_url', return_value=None)
   def test_get_oss_fuzz_latest_cov_report_info_fail(self, _, mock_error):
     """Tests that _get_oss_fuzz_latest_cov_report_info works as intended when we
     can't get latest report info."""
     result = get_coverage._get_oss_fuzz_latest_cov_report_info('project')
     self.assertIsNone(result)
     mock_error.assert_called_with(
         'Could not get the coverage report json from url: %s.',
         self.LATEST_REPORT_INFO_URL)


 if __name__ == '__main__':
   unittest.main()
	# Copyright 2021 Google LLC
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.
	"""Tests for get_coverage.py"""
	import os
	import json
	import unittest
	from unittest import mock

	from pyfakefs import fake_filesystem_unittest
	import pytest

	import get_coverage

	# pylint: disable=protected-access

	TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
	'test_data')

	PROJECT_NAME = 'curl'
	REPO_PATH = '/src/curl'
	FUZZ_TARGET = 'curl_fuzzer'
	PROJECT_COV_JSON_FILENAME = 'example_curl_cov.json'
	FUZZ_TARGET_COV_JSON_FILENAME = 'example_curl_fuzzer_cov.json'
	INVALID_TARGET = 'not-a-fuzz-target'

	with open(os.path.join(TEST_DATA_PATH,
	PROJECT_COV_JSON_FILENAME),) as cov_file_handle:
	PROJECT_COV_INFO = json.loads(cov_file_handle.read())


	class GetOssFuzzFuzzerStatsDirUrlTest(unittest.TestCase):
	"""Tests _get_oss_fuzz_fuzzer_stats_dir_url."""

	@mock.patch('http_utils.get_json_from_url',
	return_value={
	'fuzzer_stats_dir':
	'gs://oss-fuzz-coverage/systemd/fuzzer_stats/20210303'
	})
	def test_get_valid_project(self, mock_get_json_from_url):
	"""Tests that a project's coverage report can be downloaded and parsed.

	NOTE: This test relies on the PROJECT_NAME repo's coverage report.
	The "example" project was not used because it has no coverage reports.
	"""
	result = get_coverage._get_oss_fuzz_fuzzer_stats_dir_url(PROJECT_NAME)
	(url,), _ = mock_get_json_from_url.call_args
	self.assertEqual(
	'https://storage.googleapis.com/oss-fuzz-coverage/'
	'latest_report_info/curl.json', url)

	expected_result = (
	'https://storage.googleapis.com/oss-fuzz-coverage/systemd/fuzzer_stats/'
	'20210303')
	self.assertEqual(result, expected_result)

	def test_get_invalid_project(self):
	"""Tests that passing a bad project returns None."""
	self.assertIsNone(
	get_coverage._get_oss_fuzz_fuzzer_stats_dir_url('not-a-proj'))


	class OSSFuzzCoverageGetTargetCoverageTest(unittest.TestCase):
	"""Tests OSSFuzzCoverage.get_target_coverage."""

	def setUp(self):
	with mock.patch('get_coverage._get_oss_fuzz_latest_cov_report_info',
	return_value=PROJECT_COV_INFO):
	self.oss_fuzz_coverage = get_coverage.OSSFuzzCoverage(
	REPO_PATH, PROJECT_NAME)

	@mock.patch('http_utils.get_json_from_url', return_value={})
	def test_valid_target(self, mock_get_json_from_url):
	"""Tests that a target's coverage report can be downloaded and parsed."""
	self.oss_fuzz_coverage.get_target_coverage(FUZZ_TARGET)
	(url,), _ = mock_get_json_from_url.call_args
	self.assertEqual(
	'https://storage.googleapis.com/oss-fuzz-coverage/'
	'curl/fuzzer_stats/20200226/curl_fuzzer.json', url)

	def test_invalid_target(self):
	"""Tests that passing an invalid target coverage report returns None."""
	self.assertIsNone(
	self.oss_fuzz_coverage.get_target_coverage(INVALID_TARGET))

	@mock.patch('get_coverage._get_oss_fuzz_latest_cov_report_info',
	return_value=None)
	def test_invalid_project_json(self, _): # pylint: disable=no-self-use
	"""Tests an invalid project JSON results in None being returned."""
	with pytest.raises(get_coverage.CoverageError):
	get_coverage.OSSFuzzCoverage(REPO_PATH, PROJECT_NAME)


	def _get_expected_curl_covered_file_list():
	"""Returns the expected covered file list for
	FUZZ_TARGET_COV_JSON_FILENAME."""
	curl_files_list_path = os.path.join(TEST_DATA_PATH,
	'example_curl_file_list.json')
	with open(curl_files_list_path) as file_handle:
	return json.loads(file_handle.read())


	def _get_example_curl_coverage():
	"""Returns the contents of the fuzzer stats JSON file for
	FUZZ_TARGET_COV_JSON_FILENAME."""
	with open(os.path.join(TEST_DATA_PATH,
	FUZZ_TARGET_COV_JSON_FILENAME)) as file_handle:
	return json.loads(file_handle.read())


	class OSSFuzzCoverageGetFilesCoveredByTargetTest(unittest.TestCase):
	"""Tests OSSFuzzCoverage.get_files_covered_by_target."""

	def setUp(self):
	with mock.patch('get_coverage._get_oss_fuzz_latest_cov_report_info',
	return_value=PROJECT_COV_INFO):
	self.oss_fuzz_coverage = get_coverage.OSSFuzzCoverage(
	REPO_PATH, PROJECT_NAME)

	def test_valid_target(self):
	"""Tests that covered files can be retrieved from a coverage report."""
	fuzzer_cov_data = _get_example_curl_coverage()
	with mock.patch('get_coverage.OSSFuzzCoverage.get_target_coverage',
	return_value=fuzzer_cov_data):
	file_list = self.oss_fuzz_coverage.get_files_covered_by_target(
	FUZZ_TARGET)

	expected_file_list = _get_expected_curl_covered_file_list()
	self.assertCountEqual(file_list, expected_file_list)

	def test_invalid_target(self):
	"""Tests passing invalid fuzz target returns None."""
	self.assertIsNone(
	self.oss_fuzz_coverage.get_files_covered_by_target(INVALID_TARGET))


	class FilesystemCoverageGetFilesCoveredByTargetTest(
	fake_filesystem_unittest.TestCase):
	"""Tests FilesystemCoverage.get_files_covered_by_target."""

	def setUp(self):
	_fuzzer_cov_data = _get_example_curl_coverage()
	self._expected_file_list = _get_expected_curl_covered_file_list()
	self.coverage_path = '/coverage'
	self.filesystem_coverage = get_coverage.FilesystemCoverage(
	REPO_PATH, self.coverage_path)
	self.setUpPyfakefs()
	self.fs.create_file(os.path.join(self.coverage_path, 'fuzzer_stats',
	FUZZ_TARGET + '.json'),
	contents=json.dumps(_fuzzer_cov_data))

	def test_valid_target(self):
	"""Tests that covered files can be retrieved from a coverage report."""
	file_list = self.filesystem_coverage.get_files_covered_by_target(
	FUZZ_TARGET)
	self.assertCountEqual(file_list, self._expected_file_list)

	def test_invalid_target(self):
	"""Tests passing invalid fuzz target returns None."""
	self.assertIsNone(
	self.filesystem_coverage.get_files_covered_by_target(INVALID_TARGET))


	class IsFileCoveredTest(unittest.TestCase):
	"""Tests for is_file_covered."""

	def test_is_file_covered_covered(self):
	"""Tests that is_file_covered returns True for a covered file."""
	file_coverage = {
	'filename': '/src/systemd/src/basic/locale-util.c',
	'summary': {
	'regions': {
	'count': 204,
	'covered': 200,
	'notcovered': 200,
	'percent': 98.03
	}
	}
	}
	self.assertTrue(get_coverage.is_file_covered(file_coverage))

	def test_is_file_covered_not_covered(self):
	"""Tests that is_file_covered returns False for a not covered file."""
	file_coverage = {
	'filename': '/src/systemd/src/basic/locale-util.c',
	'summary': {
	'regions': {
	'count': 204,
	'covered': 0,
	'notcovered': 0,
	'percent': 0
	}
	}
	}
	self.assertFalse(get_coverage.is_file_covered(file_coverage))


	class GetOssFuzzLatestCovReportInfo(unittest.TestCase):
	"""Tests that _get_oss_fuzz_latest_cov_report_info works as
	intended."""

	PROJECT = 'project'
	LATEST_REPORT_INFO_URL = ('https://storage.googleapis.com/oss-fuzz-coverage/'
	'latest_report_info/project.json')

	@mock.patch('logging.error')
	@mock.patch('http_utils.get_json_from_url', return_value={'coverage': 1})
	def test_get_oss_fuzz_latest_cov_report_info(self, mock_get_json_from_url,
	mock_error):
	"""Tests that _get_oss_fuzz_latest_cov_report_info works as intended."""
	result = get_coverage._get_oss_fuzz_latest_cov_report_info(self.PROJECT)
	self.assertEqual(result, {'coverage': 1})
	mock_error.assert_not_called()
	mock_get_json_from_url.assert_called_with(self.LATEST_REPORT_INFO_URL)

	@mock.patch('logging.error')
	@mock.patch('http_utils.get_json_from_url', return_value=None)
	def test_get_oss_fuzz_latest_cov_report_info_fail(self, _, mock_error):
	"""Tests that _get_oss_fuzz_latest_cov_report_info works as intended when we
	can't get latest report info."""
	result = get_coverage._get_oss_fuzz_latest_cov_report_info('project')
	self.assertIsNone(result)
	mock_error.assert_called_with(
	'Could not get the coverage report json from url: %s.',
	self.LATEST_REPORT_INFO_URL)


	if __name__ == '__main__':
	unittest.main()