tools/code_coverage/coverage_test.py

#!/usr/bin/env python
# Copyright 2018 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Tests for code coverage tools."""

import os
import re
import shutil
import subprocess
import sys
import unittest
import coverage_utils


def _RecursiveDirectoryListing(dirpath):
  """Returns a list of relative paths to all files in a given directory."""
  result = []
  for root, _, files in os.walk(dirpath):
    for f in files:
      result.append(os.path.relpath(os.path.join(root, f), dirpath))
  return result


def _ReadFile(filepath):
  """Returns contents of a given file."""
  with open(filepath) as f:
    return f.read()


class CoverageTest(unittest.TestCase):

  def setUp(self):
    self.maxDiff = 1000
    self.COVERAGE_TOOLS_DIR = os.path.abspath(os.path.dirname(__file__))
    self.COVERAGE_SCRIPT = os.path.join(self.COVERAGE_TOOLS_DIR, 'coverage.py')
    self.COVERAGE_UTILS = os.path.join(self.COVERAGE_TOOLS_DIR,
                                       'coverage_utils.py')

    self.CHROMIUM_SRC_DIR = os.path.dirname(
        os.path.dirname(self.COVERAGE_TOOLS_DIR))
    self.BUILD_DIR = os.path.join(self.CHROMIUM_SRC_DIR, 'out',
                                  'code_coverage_tools_test')

    self.REPORT_DIR_1 = os.path.join(self.BUILD_DIR, 'report1')
    self.REPORT_DIR_1_NO_COMPONENTS = os.path.join(self.BUILD_DIR,
                                                   'report1_no_components')
    self.REPORT_DIR_2 = os.path.join(self.BUILD_DIR, 'report2')
    self.REPORT_DIR_3 = os.path.join(self.BUILD_DIR, 'report3')
    self.REPORT_DIR_4 = os.path.join(self.BUILD_DIR, 'report4')

    self.LLVM_COV = os.path.join(self.CHROMIUM_SRC_DIR, 'third_party',
                                 'llvm-build', 'Release+Asserts', 'bin',
                                 'llvm-cov')

    self.PYTHON = 'python3'
    self.PLATFORM = coverage_utils.GetHostPlatform()
    if self.PLATFORM == 'win32':
      self.LLVM_COV += '.exe'
      self.PYTHON += '.exe'

    # Even though 'is_component_build=false' is recommended, we intentionally
    # use 'is_component_build=true' to test handling of shared libraries.
    self.GN_ARGS = ('use_clang_coverage=true '
                    'dcheck_always_on=true '
                    'ffmpeg_branding=\"ChromeOS\" '
                    'is_component_build=true '
                    'is_debug=false '
                    'proprietary_codecs=true '
                    'use_libfuzzer=true')

    shutil.rmtree(self.BUILD_DIR, ignore_errors=True)

    gn_gen_cmd = ['gn', 'gen', self.BUILD_DIR, '--args=%s' % self.GN_ARGS]
    self.run_cmd(gn_gen_cmd)

    build_cmd = [
        'autoninja', '-C', self.BUILD_DIR, 'crypto_unittests',
        'libpng_read_fuzzer'
    ]
    self.run_cmd(build_cmd)

  def tearDown(self):
    shutil.rmtree(self.BUILD_DIR, ignore_errors=True)

  def run_cmd(self, cmd):
    return subprocess.check_output(cmd, cwd=self.CHROMIUM_SRC_DIR)

  def verify_component_view(self, filepath):
    """Asserts that a given component view looks correct."""
    # There must be several Blink and Internals components.
    with open(filepath) as f:
      data = f.read()

    counts = data.count('Blink') + data.count('Internals')
    self.assertGreater(counts, 5)

  def verify_directory_view(self, filepath):
    """Asserts that a given directory view looks correct."""
    # Directory view page does a redirect to another page, extract its URL.
    with open(filepath) as f:
      data = f.read()

    url = re.search(r'.*refresh.*url=([a-zA-Z0-9_\-\/.]+).*', data).group(1)
    directory_view_path = os.path.join(os.path.dirname(filepath), url)

    # There must be at least 'crypto' and 'third_party' directories.
    with open(directory_view_path) as f:
      data = f.read()

    self.assertTrue('crypto' in data and 'third_party' in data)

  def verify_file_view(self, filepath):
    """Asserts that a given file view looks correct."""
    # There must be hundreds of '.*crypto.*' files and 10+ of '.*libpng.*'.
    with open(filepath) as f:
      data = f.read()

    self.assertGreater(data.count('crypto'), 100)
    self.assertGreater(data.count('libpng'), 10)

  def verify_lcov_file(self, filepath):
    """Asserts that a given lcov file looks correct."""
    with open(filepath) as f:
      data = f.read()

    self.assertGreater(data.count('SF:'), 100)
    self.assertGreater(data.count('crypto'), 100)
    self.assertGreater(data.count('libpng'), 10)

  def test_different_workflows_and_cross_check_the_results(self):
    """Test a few different workflows and assert that the results are the same

    and look legit.
    """
    # Testcase 1. End-to-end report generation using coverage.py script. This is
    # the workflow of a regular user.
    cmd = [
        self.COVERAGE_SCRIPT,
        'crypto_unittests',
        'libpng_read_fuzzer',
        '-v',
        '-b',
        self.BUILD_DIR,
        '-o',
        self.REPORT_DIR_1,
        '-c'
        '%s/crypto_unittests' % self.BUILD_DIR,
        '-c',
        '%s/libpng_read_fuzzer -runs=0 third_party/libpng/' % self.BUILD_DIR,
    ]
    self.run_cmd(cmd)

    output_dir = os.path.join(self.REPORT_DIR_1, self.PLATFORM)
    self.verify_component_view(
        os.path.join(output_dir, 'component_view_index.html'))
    self.verify_directory_view(
        os.path.join(output_dir, 'directory_view_index.html'))
    self.verify_file_view(os.path.join(output_dir, 'file_view_index.html'))

    # Also try generating a report without components view. Useful for cross
    # checking with the report produced in the testcase #3.
    cmd = [
        self.COVERAGE_SCRIPT,
        'crypto_unittests',
        'libpng_read_fuzzer',
        '-v',
        '-b',
        self.BUILD_DIR,
        '-o',
        self.REPORT_DIR_1_NO_COMPONENTS,
        '-c'
        '%s/crypto_unittests' % self.BUILD_DIR,
        '-c',
        '%s/libpng_read_fuzzer -runs=0 third_party/libpng/' % self.BUILD_DIR,
        '--no-component-view',
    ]
    self.run_cmd(cmd)

    output_dir = os.path.join(self.REPORT_DIR_1_NO_COMPONENTS, self.PLATFORM)
    self.verify_directory_view(
        os.path.join(output_dir, 'directory_view_index.html'))
    self.verify_file_view(os.path.join(output_dir, 'file_view_index.html'))
    self.assertFalse(
        os.path.exists(os.path.join(output_dir, 'component_view_index.html')))

    # Testcase #2. Run the script for post processing in Chromium tree. This is
    # the workflow of the code coverage bots.
    instr_profile_path = os.path.join(self.REPORT_DIR_1, self.PLATFORM,
                                      'coverage.profdata')

    cmd = [
        self.COVERAGE_SCRIPT,
        'crypto_unittests',
        'libpng_read_fuzzer',
        '-v',
        '-b',
        self.BUILD_DIR,
        '-p',
        instr_profile_path,
        '-o',
        self.REPORT_DIR_2,
    ]
    self.run_cmd(cmd)

    # Verify that the output dirs are the same except of the expected diff.
    report_1_listing = set(_RecursiveDirectoryListing(self.REPORT_DIR_1))
    report_2_listing = set(_RecursiveDirectoryListing(self.REPORT_DIR_2))
    logs_subdir = os.path.join(self.PLATFORM, 'logs')
    self.assertEqual(
        set([
            os.path.join(self.PLATFORM, 'coverage.profdata'),
            os.path.join(logs_subdir, 'crypto_unittests_output.log'),
            os.path.join(logs_subdir, 'libpng_read_fuzzer_output.log'),
        ]), report_1_listing - report_2_listing)

    output_dir = os.path.join(self.REPORT_DIR_2, self.PLATFORM)
    self.verify_component_view(
        os.path.join(output_dir, 'component_view_index.html'))
    self.verify_directory_view(
        os.path.join(output_dir, 'directory_view_index.html'))
    self.verify_file_view(os.path.join(output_dir, 'file_view_index.html'))

    # Verify that the file view pages are binary equal.
    report_1_file_view_data = _ReadFile(
        os.path.join(self.REPORT_DIR_1, self.PLATFORM, 'file_view_index.html'))
    report_2_file_view_data = _ReadFile(
        os.path.join(self.REPORT_DIR_2, self.PLATFORM, 'file_view_index.html'))
    self.assertEqual(report_1_file_view_data, report_2_file_view_data)

    # Testcase #3, run coverage_utils.py on manually produced report and summary
    # file. This is the workflow of OSS-Fuzz code coverage job.
    objects = [
        '-object=%s' % os.path.join(self.BUILD_DIR, 'crypto_unittests'),
        '-object=%s' % os.path.join(self.BUILD_DIR, 'libpng_read_fuzzer'),
    ]

    cmd = [
        self.PYTHON,
        self.COVERAGE_UTILS,
        '-v',
        'shared_libs',
        '-build-dir=%s' % self.BUILD_DIR,
    ] + objects

    shared_libraries = self.run_cmd(cmd)
    objects.extend(shared_libraries.split())

    instr_profile_path = os.path.join(self.REPORT_DIR_1_NO_COMPONENTS,
                                      self.PLATFORM, 'coverage.profdata')
    cmd = [
        self.LLVM_COV,
        'show',
        '-format=html',
        '-output-dir=%s' % self.REPORT_DIR_3,
        '-instr-profile=%s' % instr_profile_path,
    ] + objects
    if self.PLATFORM in ['linux', 'mac']:
      cmd.extend(['-Xdemangler', 'c++filt', '-Xdemangler', '-n'])
    self.run_cmd(cmd)

    cmd = [
        self.LLVM_COV,
        'export',
        '-summary-only',
        '-instr-profile=%s' % instr_profile_path,
    ] + objects
    summary_output = self.run_cmd(cmd)

    summary_path = os.path.join(self.REPORT_DIR_3, 'summary.json')
    with open(summary_path, 'wb') as f:
      f.write(summary_output)

    cmd = [
        self.PYTHON,
        self.COVERAGE_UTILS,
        '-v',
        'post_process',
        '-src-root-dir=%s' % self.CHROMIUM_SRC_DIR,
        '-summary-file=%s' % summary_path,
        '-output-dir=%s' % self.REPORT_DIR_3,
    ]
    self.run_cmd(cmd)

    output_dir = os.path.join(self.REPORT_DIR_3, self.PLATFORM)
    self.verify_directory_view(
        os.path.join(output_dir, 'directory_view_index.html'))
    self.verify_file_view(os.path.join(output_dir, 'file_view_index.html'))
    self.assertFalse(
        os.path.exists(os.path.join(output_dir, 'component_view_index.html')))

    # Verify that the file view pages are binary equal.
    report_1_file_view_data_no_component = _ReadFile(
        os.path.join(self.REPORT_DIR_1_NO_COMPONENTS, self.PLATFORM,
                     'file_view_index.html'))
    report_3_file_view_data = _ReadFile(
        os.path.join(self.REPORT_DIR_3, self.PLATFORM, 'file_view_index.html'))
    self.assertEqual(report_1_file_view_data_no_component,
                     report_3_file_view_data)

    # Testcase 4. Export coverage data in lcov format using coverage.py script.
    cmd = [
        self.COVERAGE_SCRIPT,
        'crypto_unittests',
        'libpng_read_fuzzer',
        '--format',
        'lcov',
        '-v',
        '-b',
        self.BUILD_DIR,
        '-o',
        self.REPORT_DIR_4,
        '-c'
        '%s/crypto_unittests' % self.BUILD_DIR,
        '-c',
        '%s/libpng_read_fuzzer -runs=0 third_party/libpng/' % self.BUILD_DIR,
    ]
    self.run_cmd(cmd)

    output_dir = os.path.join(self.REPORT_DIR_4, self.PLATFORM)
    self.verify_lcov_file(os.path.join(output_dir, 'coverage.lcov'))


if __name__ == '__main__':
  unittest.main()