Support fuzz_targets property in clusterfuzz_manifest.json

Author: notvictorl

src/clusterfuzz/_internal/build_management/build_archive.py

@@ -141,17 +141,17 @@ class DefaultBuildArchive(BuildArchive):
 
   def __init__(self, reader: archive.ArchiveReader):
     super().__init__(reader)
-    self._fuzz_targets = {}
+    self._fuzz_targets = None
 
   def get_path_for_target(self, fuzz_target: str) -> str:
     """Returns the path in the archive of the fuzz_target if found.
     This is needed because target name normalization means we're losing
     information about the actual file reprensenting the fuzz_target.
     """
-    if not self._fuzz_targets:
+    if self._fuzz_targets is None:
       self.list_fuzz_targets()
 
-    if not fuzz_target in self._fuzz_targets:
+    if not self._fuzz_targets or fuzz_target not in self._fuzz_targets:
       return None
 
     return self._fuzz_targets[fuzz_target]
@@ -185,8 +185,10 @@ def get_target_dependencies(
     return to_extract
 
   def list_fuzz_targets(self) -> List[str]:
-    if self._fuzz_targets:
+    if self._fuzz_targets is not None:
       return list(self._fuzz_targets.keys())
+
+    self._fuzz_targets = {}
     # Import here as this path is not available in App Engine context.
     from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils
 
@@ -304,6 +306,17 @@ def __init__(self,
             'clusterfuzz_manifest.json was incorrectly formatted or missing an '
             'archive_schema_version field')
         self._archive_schema_version = default_archive_schema_version
+
+      fuzz_targets_data = manifest.get('fuzz_targets')
+      if isinstance(fuzz_targets_data, list):
+        self._fuzz_targets = {}
+        from clusterfuzz._internal.bot.fuzzers import utils as fuzzer_utils
+        for target_path in fuzz_targets_data:
+          if isinstance(target_path, str):
+            fuzz_target = fuzzer_utils.normalize_target_name(target_path)
+            self._fuzz_targets[fuzz_target] = target_path
+      elif fuzz_targets_data is not None:
+        logs.error('fuzz_targets in clusterfuzz_manifest.json is not a list')
     else:
       self._archive_schema_version = default_archive_schema_version
 

src/clusterfuzz/_internal/build_management/build_manager.py

@@ -522,6 +522,19 @@ def _unpack_build(self,
       with self._open_build_archive(base_build_dir, build_dir, build_url,
                                     http_build_url) as build:
         unpack_start_time = time.time()
+        if environment.is_engine_fuzzer_job() and not self.fuzz_target:
+          # If no fuzz target is selected (e.g. during initial fuzz target
+          # discovery), we only need the list of fuzz targets. We get them
+          # from the build archive (which instantly reads
+          # clusterfuzz_manifest.json if present) and skip unpacking the
+          # archive to disk entirely to prevent out of disk space errors.
+          list_fuzz_target_start_time = time.time()
+          self._fuzz_targets = list(build.list_fuzz_targets())
+          _emit_job_build_retrieval_metric(list_fuzz_target_start_time,
+                                           'list_fuzz_targets',
+                                           self._build_type)
+          logs.info('No fuzz target selected; skipping unpack for discovery.')
+          return True
         if not self._unpack_everything:
           # We will never unpack the full build so we need to get the targets
           # from the build archive.
@@ -1351,7 +1364,8 @@ def setup_regular_build(revision,
         build.build_dir,  # Store inside the main build.
         revision,
         extra_build_url,
-        build_prefix=fuzzer_utils.EXTRA_BUILD_DIR)
+        build_prefix=fuzzer_utils.EXTRA_BUILD_DIR,
+        fuzz_target=fuzz_target)
     if not build.setup():
       return None
 

src/clusterfuzz/_internal/tests/core/bot/testcase_manager_test.py

@@ -904,7 +904,7 @@ def test_reproduce(self):
 
     self._setup_env(job_type='libfuzzer_asan_job')
 
-    build_manager.setup_build()
+    build_manager.setup_build(fuzz_target='test_fuzzer')
     result = testcase_manager.engine_reproduce(
         libfuzzer_engine.Engine(), 'test_fuzzer', testcase_file_path, [], 30)
 

src/clusterfuzz/_internal/tests/core/build_management/build_archive_test.py

@@ -409,3 +409,58 @@ def test_manifest_is_correctly_read(self):
     self._generate_manifest(1)
     test_archive = build_archive.ChromeBuildArchive(self.mock.open.return_value)
     self.assertEqual(test_archive.archive_schema_version(), 1)
+
+  def _generate_manifest_with_fuzz_targets(self, archive_schema_version,
+                                           fuzz_targets):
+    """Mocks open calls so that they return a buffer containing valid JSON for
+    the given archive schema version and fuzz_targets."""
+
+    def _mock_open(_):
+      buffer = io.BytesIO(b'')
+      buffer.write(
+          json.dumps({
+              'archive_schema_version': archive_schema_version,
+              'fuzz_targets': fuzz_targets,
+          }).encode())
+      buffer.seek(0)
+      return buffer
+
+    self.mock.open.return_value.open.side_effect = _mock_open
+
+  def test_manifest_fuzz_targets_list(self):
+    """Tests that fuzz_targets specified as a list in the manifest correctly
+    populates self._fuzz_targets and bypasses discovery."""
+    self.mock.file_exists.return_value = True
+    self._generate_manifest_with_fuzz_targets(
+        1, ['out/build/my_fuzzer', 'out/build/other_fuzzer'])
+    test_archive = build_archive.ChromeBuildArchive(self.mock.open.return_value)
+    self.assertEqual(test_archive.archive_schema_version(), 1)
+    self.assertCountEqual(test_archive.list_fuzz_targets(),
+                          ['my_fuzzer', 'other_fuzzer'])
+    self.assertEqual(
+        test_archive.get_path_for_target('my_fuzzer'), 'out/build/my_fuzzer')
+    # Ensure list_members was never called for fuzz target discovery.
+    self.mock.open.return_value.list_members.assert_not_called()
+
+  def test_manifest_fuzz_targets_invalid(self):
+    """Tests that invalid fuzz_targets (e.g. dict) in the manifest are ignored
+    and we fallback to discovery."""
+    self.mock.file_exists.return_value = True
+    self._generate_manifest_with_fuzz_targets(
+        1, {'my_fuzzer': 'out/build/my_fuzzer'})
+    self.mock.open.return_value.list_members.return_value = []
+    test_archive = build_archive.ChromeBuildArchive(self.mock.open.return_value)
+    self.assertEqual(test_archive.archive_schema_version(), 1)
+    test_archive.list_fuzz_targets()
+    self.mock.open.return_value.list_members.assert_called_once()
+
+  def test_manifest_fuzz_targets_missing(self):
+    """Tests that missing fuzz_targets in the manifest are ignored
+    and we fallback to discovery."""
+    self.mock.file_exists.return_value = True
+    self._generate_manifest(1)
+    self.mock.open.return_value.list_members.return_value = []
+    test_archive = build_archive.ChromeBuildArchive(self.mock.open.return_value)
+    self.assertEqual(test_archive.archive_schema_version(), 1)
+    test_archive.list_fuzz_targets()
+    self.mock.open.return_value.list_members.assert_called_once()

src/clusterfuzz/_internal/tests/core/build_management/build_manager_test.py

@@ -532,6 +532,26 @@ def _assert_env_vars(self):
         os.environ['BUILD_DIR'],
         '/builds/path_be4c9ca0267afcd38b7c1a3eebb5998d0908f025/revisions')
 
+  def test_setup_fuzz_discovery(self):
+    """Tests setting up a build during initial fuzz target discovery.
+
+    (fuzz_target=None).
+    """
+    os.environ['TASK_NAME'] = 'fuzz'
+    self.mock.time.return_value = 1000.0
+    build = build_manager.setup_regular_build(2, fuzz_target=None)
+    self.assertIsInstance(build, build_manager.RegularBuild)
+    self.assertEqual(_get_timestamp(build.base_build_dir), 1000.0)
+
+    self._assert_env_vars()
+    self.assertEqual(os.environ['APP_REVISION'], '2')
+
+    # Verify unpack() was never called because we skipped unpacking
+    # for discovery.
+    unpack_mock = self.mock.open.return_value.__enter__.return_value.unpack
+    self.assertEqual(unpack_mock.call_count, 0)
+    self.assertCountEqual(['target1', 'target2', 'target3'], build.fuzz_targets)
+
   @parameterized.parameterized.expand(['True', 'False'])
   def test_setup_fuzz(self, unpack_all):
     """Tests setting up a build during fuzzing."""
@@ -741,7 +761,9 @@ def test_setup_fuzz_over_http(self):
     os.environ['ALLOW_UNPACK_OVER_HTTP'] = "True"
     self.mock.unzip_over_http_compatible.return_value = True
     self.mock.time.return_value = 1000.0
-    build = build_manager.setup_regular_build(2)
+    fuzz_target = build_manager.pick_random_fuzz_target(
+        target_weights=self.target_weights)
+    build = build_manager.setup_regular_build(2, fuzz_target=fuzz_target)
     self.assertIsInstance(build, build_manager.RegularBuild)
     self.assertEqual(_get_timestamp(build.base_build_dir), 1000.0)
 
@@ -756,7 +778,7 @@ def test_setup_fuzz_over_http(self):
 
     # Test setting up build again.
     self.mock.time.return_value = 1005.0
-    build = build_manager.setup_regular_build(2)
+    build = build_manager.setup_regular_build(2, fuzz_target=fuzz_target)
 
     self.assertIsInstance(build, build_manager.RegularBuild)
 
@@ -778,7 +800,9 @@ def test_setup_fuzz_over_http_unpack_all(self):
         'https://example.com/path/file-release-([0-9]+).zip')
     self.mock.unzip_over_http_compatible.return_value = True
     self.mock.time.return_value = 1000.0
-    build = build_manager.setup_regular_build(2)
+    fuzz_target = build_manager.pick_random_fuzz_target(
+        target_weights=self.target_weights)
+    build = build_manager.setup_regular_build(2, fuzz_target=fuzz_target)
     self.assertIsInstance(build, build_manager.RegularBuild)
     self.assertEqual(_get_timestamp(build.base_build_dir), 1000.0)
 
@@ -791,7 +815,7 @@ def test_setup_fuzz_over_http_unpack_all(self):
     # Test setting up build again.
     os.environ['FUZZ_TARGET'] = ''
     self.mock.time.return_value = 1005.0
-    build = build_manager.setup_regular_build(2)
+    build = build_manager.setup_regular_build(2, fuzz_target=fuzz_target)
 
     self.assertIsInstance(build, build_manager.RegularBuild)