tools/perf/contrib/orderfile/orderfile.py

# Copyright 2018 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
""" Orderfile Generation and Testing

This provides a suite of benchmarks for orderfile generation and testing based
on the mobile system health benchmarks.

The orderfile_generation.training benchmark is meant to be run with an
orderfile instrumented build to produce function usage profiles. It can be
invoked as follows:
    (CHROMIUM_OUTPUT_DIR=out/Instrumented; \
     ./tools/perf/run_benchmark --device=${YOUR_DEVICE_SN} --browser=exact \
        --browser-executable=${CHROMIUM_OUTPUT_DIR}/apks/Monochrome.apk \
        orderfile_generation.training)

The orderfile_generation.testing benchmark has a smaller test set whose
benchmarks are distinct from those in orderfile_generation.training. Run it as
follows, note the --orderfile-memory-optimization flag is necessary only with
legacy orderfile builds of chrome.
    (CHROMIUM_OUTPUT_DIR=out/Official; \
     ./tools/perf/run_benchmark --device=${YOUR_DEVICE_SN} --browser=exact \
        --browser-executable=${CHROMIUM_OUTPUT_DIR}/apks/Monochrome.apk \
        --extra-browser-args=--orderfile-memory-optimization \
        --results-label=${LABEL_CORRESPONDING_TO_YOUR_BUILD} \
        orderfile_generation.training)

The orderfile_generation.variation.* benchmarks use a smaller training set so
that several test set variations can be produced. They are run as above but
using the following benchmark names.
    orderfile_generation.variation.training
    orderfile_generation.variation.testing0
    orderfile_generation.variation.testing1
    orderfile_generation.variation.testing2

The orderfile_generation.debugging benchmark is a short benchmark of 3 stories
that is useful for debugging hardware and test setup problems.
"""

import random

from benchmarks import system_health, speedometer3
from page_sets.system_health import platforms
from page_sets.system_health import system_health_stories
from telemetry import benchmark
from telemetry import story
from telemetry.timeline import chrome_trace_category_filter
from telemetry.timeline import chrome_trace_config
from telemetry.web_perf import timeline_based_measurement


class OrderfileStorySet(story.StorySet):
  """User stories for orderfile generation.

  The run set specified in the constructor splits the stories into training and
  testing sets (or debugging, see the code for details).
  """
  # Run set names.
  TRAINING = 'training'
  TESTING = 'testing'
  DEBUGGING = 'debugging'

  _PLATFORM = 'mobile'

  _BLOCKLIST = set([
      # 0% success rate on arm (measured 2024 June).
      'browse:chrome:newtab:2019',
      # 0% success rate on arm (measured 2024 June).
      'browse:chrome:omnibox:2019',
      # 0% success rate on arm64, 2% on arm (measured 2024 June).
      'browse:news:globo:2019',
      # 35% success rate on arm64, 64% on arm (measured 2024 June).
      'browse:news:washingtonpost:2019',
      # 1% success rate on arm64, 17% on arm (measured 2024 June).
      'browse:shopping:amazon:2019',
      # Carried over from previous blocklist.
      'browse:tech:discourse_infinite_scroll:2018',
      # Carried over from previous blocklist.
      'long_running:tools:gmail-background',
      # Carried over from previous blocklist.
      'long_running:tools:gmail-foreground',
  ])

  # The random seed used for reproducible runs.
  SEED = 8675309

  # These defaults are current best practice for production orderfiles.
  DEFAULT_TRAINING = 25
  DEFAULT_TESTING = 8
  DEFAULT_VARIATIONS = 1

  # The number of variations to use with the variation benchmarks. If this is
  # changed, the number of OrderfileVariationTesting* classes declared below
  # should change as well.
  NUM_VARIATION_BENCHMARKS = 3

  def __init__(self, run_set, num_training=DEFAULT_TRAINING,
               num_testing=DEFAULT_TESTING, num_variations=DEFAULT_VARIATIONS,
               test_variation=0):
    """Create an orderfile training or testing benchmark set.

    Args:
      run_set: one of TRAINING, TESTING or DEBUGGING.
      num_training: the number of benchmarks to use for training.
      num_testing: the number of benchmarks to use in each test set.
      num_variations: the number of test set variations.
      test_variation: the test set variation to use.
    """
    super(OrderfileStorySet, self).__init__(
        archive_data_file=('../../page_sets/data/system_health_%s.json' %
                           self._PLATFORM),
        cloud_storage_bucket=story.PARTNER_BUCKET)

    assert self._PLATFORM in platforms.ALL_PLATFORMS, '{} not in {}'.format(
        self._PLATFORM, str(platforms.ALL_PLATFORMS))
    assert run_set in (self.TRAINING, self.TESTING, self.DEBUGGING)
    assert 0 <= test_variation < num_variations

    self._run_set = run_set
    self._num_training = num_training
    self._num_testing = num_testing
    self._num_variations = num_variations
    self._test_variation = test_variation

    # We want the story selection to be consistent across runs.
    random.seed(self.SEED)

    for story_class in self.RunSetStories():
      # pylint: disable=E1102
      self.AddStory(story_class(self, take_memory_measurement=True))

  def RunSetStories(self):
    possible_stories = [
        s for s in system_health_stories.IterAllSystemHealthStoryClasses()
        if (s.NAME not in self._BLOCKLIST and not s.ABSTRACT_STORY
            and self._PLATFORM in s.SUPPORTED_PLATFORMS)
    ]
    assert (self._num_training + self._num_variations * self._num_testing
            <= len(possible_stories)), \
        'We only have {} stories to work with, but want {} + {}*{}'.format(
            len(possible_stories), self._num_training, self._num_variations,
            self._num_testing)

    if self._run_set == self.DEBUGGING:
      return random.sample(possible_stories, 3)

    random.shuffle(possible_stories)
    if self._run_set == self.TRAINING:
      return possible_stories[:self._num_training]
    if self._run_set == self.TESTING:
      return possible_stories[
          (self._num_training + self._test_variation * self._num_testing):
          (self._num_training + (self._test_variation + 1) * self._num_testing)]
    assert False, 'Bad run set {}'.format(self._run_set)
    return None


class _OrderfileBenchmark(system_health.MobileMemorySystemHealth):
  """Base benchmark for orderfile generation."""
  STORY_RUN_SET = None  # Must be set in subclasses.

  def CreateStorySet(self, options):
    return OrderfileStorySet(run_set=self.STORY_RUN_SET)


# pylint: disable=R0901
@benchmark.Owner(emails=['mattcary@chromium.org'])
class OrderfileTraining(_OrderfileBenchmark):
  STORY_RUN_SET = OrderfileStorySet.TRAINING

  options = {'pageset_repeat': 2}

  @classmethod
  def Name(cls):
    return 'orderfile_generation.training'


# pylint: disable=R0901
@benchmark.Owner(emails=['mattcary@chromium.org'])
class OrderfileTesting(_OrderfileBenchmark):
  STORY_RUN_SET = OrderfileStorySet.TESTING

  options = {'pageset_repeat': 7}

  @classmethod
  def Name(cls):
    return 'orderfile_generation.testing'


class _OrderfileVariation(system_health.MobileMemorySystemHealth):
  """Orderfile generation with test set variations."""
  STORY_RUN_SET = None   # Must be set in all subclasses.
  TEST_VARIATION = 0     # Can be overridden testing subclasses.

  options = {'pageset_repeat': 7}

  def CreateStorySet(self, options):
    return OrderfileStorySet(
        run_set=self.STORY_RUN_SET,
        num_training=25, num_testing=8,
        num_variations=OrderfileStorySet.NUM_VARIATION_BENCHMARKS,
        test_variation=self.TEST_VARIATION)

  @classmethod
  def Name(cls):
    if cls.STORY_RUN_SET == OrderfileStorySet.TESTING:
      return 'orderfile_generation.variation.testing{}'.format(
          cls.TEST_VARIATION)
    if cls.STORY_RUN_SET == OrderfileStorySet.TRAINING:
      return 'orderfile_generation.variation.training'
    assert False, 'Bad STORY_RUN_SET {}'.format(cls.STORY_RUN_SET)
    return None

# pylint: disable=R0901
@benchmark.Owner(emails=['mattcary@chromium.org'])
class OrderfileVariationTraining(_OrderfileVariation):
  STORY_RUN_SET = OrderfileStorySet.TRAINING


# pylint: disable=R0901
@benchmark.Owner(emails=['mattcary@chromium.org'])
class OrderfileVariationTesting0(_OrderfileVariation):
  STORY_RUN_SET = OrderfileStorySet.TESTING
  TEST_VARIATION = 0


# pylint: disable=R0901
@benchmark.Owner(emails=['mattcary@chromium.org'])
class OrderfileVariationTesting1(_OrderfileVariation):
  STORY_RUN_SET = OrderfileStorySet.TESTING
  TEST_VARIATION = 1


# pylint: disable=R0901
@benchmark.Owner(emails=['mattcary@chromium.org'])
class OrderfileVariationTesting2(_OrderfileVariation):
  STORY_RUN_SET = OrderfileStorySet.TESTING
  TEST_VARIATION = 2


# pylint: disable=R0901
@benchmark.Owner(emails=['mattcary@chromium.org'])
class OrderfileDebugging(_OrderfileBenchmark):
  """A very short benchmark for debugging metrics collection."""
  STORY_RUN_SET = OrderfileStorySet.DEBUGGING

  options = {'pageset_repeat': 1}

  @classmethod
  def Name(cls):
    return 'orderfile_generation.debugging'

@benchmark.Owner(emails=['mattcary@chromium.org'])
class OrderfileMemory(system_health.MobileMemorySystemHealth):
  """Benchmark for native code memory footprint evaluation."""
  class OrderfileMemoryStorySet(story.StorySet):
    _STORY_SET = set(['browse:news:cnn:2021', 'browse:social:facebook:2019'])

    def __init__(self, platform, take_memory_measurement=True):
      super(OrderfileMemory.OrderfileMemoryStorySet, self).__init__(
          archive_data_file=('../../page_sets/data/system_health_%s.json' %
                             platform),
          cloud_storage_bucket=story.PARTNER_BUCKET)

      assert platform in platforms.ALL_PLATFORMS
      for story_cls in system_health_stories.IterAllSystemHealthStoryClasses():
        if (story_cls.ABSTRACT_STORY
            or platform not in story_cls.SUPPORTED_PLATFORMS
            or story_cls.NAME not in self._STORY_SET):
          continue
        self.AddStory(story_cls(self, take_memory_measurement))


  def CreateStorySet(self, options):
    return self.OrderfileMemoryStorySet(platform=self.PLATFORM)

  def CreateCoreTimelineBasedMeasurementOptions(self):
    cat_filter = chrome_trace_category_filter.ChromeTraceCategoryFilter(
        filter_string='-*,disabled-by-default-memory-infra')
    options = timeline_based_measurement.Options(cat_filter)
    # options.config.enable_android_graphics_memtrack = True
    options.SetTimelineBasedMetrics(['nativeCodeResidentMemoryMetric'])
    # Setting an empty memory dump config disables periodic dumps.
    options.config.chrome_trace_config.SetMemoryDumpConfig(
        chrome_trace_config.MemoryDumpConfig())
    return options

  @classmethod
  def Name(cls):
    return 'orderfile.memory_mobile'


@benchmark.Owner(emails=['rasikan@google.com'])
class OrderfileWebViewStartup(system_health.WebviewStartupSystemHealthBenchmark
                              ):
  """Benchmark for orderfile generation with WebView startup profiles.

  We need this class to wrap the system_health.webview_startup benchmark so
  we can apply the additional configs that trigger the devtools memory dump.
  We rely on the devtools memory dump to collect the profiles, even though
  this is a startup profile. The reason for this is so we can avoid rebuilding
  the native library between benchmarks, as the memory_mobile based benchmarks
  use this. Rebuilidng/relinking would make it difficult to merge the offsets.
  TODO(b/326927766): remove the devtools_instrumentation_dumping compiler flag
  so we can switch to startup profiling for webview startup.
  """

  def CreateStorySet(self, options):
    return system_health_stories.SystemHealthBlankStorySet(
        take_memory_measurement=True)

  def CreateCoreTimelineBasedMeasurementOptions(self):
    cat_filter = chrome_trace_category_filter.ChromeTraceCategoryFilter(
        filter_string='-*,disabled-by-default-memory-infra')
    options = timeline_based_measurement.Options(cat_filter)
    return options

  @classmethod
  def Name(cls):
    return 'orderfile_generation.webview_startup'


@benchmark.Owner(emails=['rasikan@google.com'])
class OrderfileWebViewDebugging(OrderfileWebViewStartup):
  """A very short benchmark for debugging metrics collection."""
  options = {'pageset_repeat': 1}

  @classmethod
  def Name(cls):
    return 'orderfile_generation.webview_startup_debugging'


@benchmark.Owner(emails=['rasikan@google.com'])
class OrderfileSpeedometer3(speedometer3.Speedometer3):
  enable_systrace = True
  take_memory_measurement = True
  options = {'pageset_repeat': 5}

  @classmethod
  def Name(cls):
    return 'orderfile_generation.speedometer3'


@benchmark.Owner(emails=['rasikan@google.com'])
class OrderfileSpeedometer3Debugging(OrderfileSpeedometer3):
  iteration_count = 1
  options = {'pageset_repeat': 1}

  @classmethod
  def Name(cls):
    return 'orderfile_generation.speedometer3_debugging'