Update appengine imports for external_testcase_reader. (drive by add additional component filter for extra safety)

Author: pgrace-google

src/clusterfuzz/_internal/cron/external_testcase_reader.py

@@ -21,16 +21,18 @@
 
 from appengine.libs import form
 from appengine.libs import gcs
-from appengine.libs import helpers
 from clusterfuzz._internal.config import local_config
 from clusterfuzz._internal.issue_management.google_issue_tracker import \
     issue_tracker
+from clusterfuzz._internal.metrics import logs
 
 ACCEPTED_FILETYPES = [
     'text/javascript', 'application/pdf', 'text/html', 'application/zip'
 ]
 ISSUETRACKER_ACCEPTED_STATE = 'ACCEPTED'
 ISSUETRACKER_WONTFIX_STATE = 'NOT_REPRODUCIBLE'
+UPLOAD_REQUEST_COMPONENT_FILTER = 'componentid:1600865'
+CLUSTERFUZZ_COMPONENT_FILTER = 'componentid:1457062+'
 
 
 def get_vrp_uploaders(config):
@@ -125,7 +127,7 @@ def submit_testcase(issue_id, file, filename, filetype, cmds):
   elif filetype == 'application/zip':
     job = 'linux_asan_chrome_mp'
   else:
-    raise TypeError
+    return 0
   upload_info = gcs.prepare_blob_upload()._asdict()
 
   data = {
@@ -158,18 +160,23 @@ def handle_testcases(tracker, config):
   # Handle bugs that were already submitted and still open.
   older_issues = tracker.find_issues_with_filters(
       keywords=[],
-      query_filters=['componentid:1600865', 'status:accepted'],
+      query_filters=[
+          UPLOAD_REQUEST_COMPONENT_FILTER, CLUSTERFUZZ_COMPONENT_FILTER,
+          'status:accepted'
+      ],
       only_open=True)
   for issue in older_issues:
     # Close out older bugs that may have failed to reproduce.
     if close_issue_if_not_reproducible(issue, config):
-      helpers.log('Closing issue {issue_id} as it failed to reproduce',
-                  issue.id)
+      logs.info(f'Closing issue {issue.id} as it failed to reproduce')
 
   # Handle new bugs that may need to be submitted.
   issues = tracker.find_issues_with_filters(
       keywords=[],
-      query_filters=['componentid:1600865', 'status:new'],
+      query_filters=[
+          UPLOAD_REQUEST_COMPONENT_FILTER, CLUSTERFUZZ_COMPONENT_FILTER,
+          'status:new'
+      ],
       only_open=True)
   if len(issues) == 0:
     return
@@ -189,20 +196,33 @@ def handle_testcases(tracker, config):
     reporters_map[issue.reporter] = reporters_map.get(issue.reporter, 1) + 1
     if close_issue_if_invalid(issue, attachment_metadata, commandline_flags,
                               vrp_uploaders):
-      helpers.log('Closing issue {issue_id} as it is invalid', issue.id)
+      logs.info(f'Closing issue {issue.id} as it is invalid')
       continue
 
     # Submit valid testcases.
     attachment_metadata = attachment_metadata[0]
     attachment = tracker.get_attachment(
         attachment_metadata['attachmentDataRef']['resourceName'])
-    submit_testcase(issue.id, attachment, attachment_metadata['filename'],
-                    attachment_metadata['contentType'], commandline_flags)
-    comment_message = 'Testcase submitted to clusterfuzz'
-    issue.status = ISSUETRACKER_ACCEPTED_STATE
-    issue.assignee = 'clusterfuzz@chromium.org'
+    submit_result = submit_testcase(
+        issue.id, attachment, attachment_metadata['filename'],
+        attachment_metadata['contentType'], commandline_flags)
+    if submit_result and submit_result.status_code == 200:
+      comment_message = 'Testcase submitted to clusterfuzz.'
+      issue.status = ISSUETRACKER_ACCEPTED_STATE
+      issue.assignee = 'clusterfuzz@chromium.org'
+      logs.info(f'Submitted testcase file for issue {issue.id}')
+    elif submit_result and submit_result.status_code != 200:
+      comment_message = 'Testcase submission failed during upload.'
+      issue.status = ISSUETRACKER_WONTFIX_STATE
+      logs.info(f'Testcase submission failed with error code'
+                f'{submit_result.status_code} for issue: {issue.id}')
+    else:
+      comment_message = (
+          'Testcase submission failed due to unrecognized poc type.')
+      issue.status = ISSUETRACKER_WONTFIX_STATE
+      logs.info(f'Testcase submission failed due to poc type error'
+                f'for issue: {issue.id}')
     issue.save(new_comment=comment_message, notify=True)
-    helpers.log('Submitted testcase file for issue {issue_id}', issue.id)
 
 
 def main():

src/clusterfuzz/_internal/tests/appengine/handlers/cron/external_testcase_reader_test.py

@@ -129,6 +129,53 @@ def test_handle_testcases_no_issues(self, mock_close_issue_if_invalid,
     mock_submit_testcase.assert_not_called()
 
 
+@mock.patch.object(
+    external_testcase_reader.gcs, 'prepare_blob_upload', autospec=True)
+@mock.patch.object(
+    external_testcase_reader.form, 'generate_csrf_token', autospec=True)
+class ExternalTestcaseReaderSubmitTestcaseTest(unittest.TestCase):
+  """external_testcase_reader submit_testcase tests."""
+
+  @mock.patch.object(external_testcase_reader.requests, 'post', autospec=True)
+  def test_submit_valid_testcase(self, mock_post, mock_csrf, mock_blob):
+    """Test a basic submit_testcase with a valid testcase."""
+    html_file = mock.MagicMock()
+    actual = external_testcase_reader.submit_testcase(
+        123, html_file, 'filename', 'text/html', ['command1'])
+
+    self.assertNotEqual(0, actual)
+    mock_post.assert_called()
+    mock_csrf.assert_called()
+    mock_blob.assert_called()
+
+  @mock.patch.object(external_testcase_reader.requests, 'post', autospec=True)
+  def test_submit_unsupported_testcase(self, mock_post, mock_csrf, mock_blob):
+    """Test a basic submit_testcase with an unsupported testcase."""
+    some_file = mock.MagicMock()
+    actual = external_testcase_reader.submit_testcase(
+        123, some_file, 'filename', 'invalid/type', ['command1'])
+
+    self.assertEqual(0, actual)
+    mock_post.assert_not_called()
+    mock_csrf.assert_not_called()
+    mock_blob.assert_not_called()
+
+  @mock.patch.object(external_testcase_reader.requests, 'post', autospec=True)
+  def test_submit_post_failure(self, mock_post, mock_csrf, mock_blob):
+    """Test a basic submit_testcase with an http post failure."""
+    invalid_file = mock.MagicMock()
+    invalid_response = mock.MagicMock()
+    invalid_response.status_code = 400
+    mock_post.return_value = invalid_response
+    actual = external_testcase_reader.submit_testcase(
+        123, invalid_file, 'filename', 'text/html', ['command1'])
+
+    self.assertEqual(400, actual.status_code)
+    mock_post.assert_called()
+    mock_csrf.assert_called()
+    mock_blob.assert_called()
+
+
 class ExternalTestcaseReaderInvalidIssueTest(unittest.TestCase):
   """external_testcase_reader close_issue_if_invalid tests."""
 

src/python/bot/startup/run_cron.py

@@ -49,15 +49,19 @@ def main():
   if os.path.exists(config_modules_path):
     sys.path.append(config_modules_path)
 
+  task = sys.argv[1]
+  if task == 'external_testcase_reader':
+    appengine_path = os.path.join(root_directory, 'src', 'appengine')
+    if os.path.exists(appengine_path):
+      sys.path.append(appengine_path)
+
   try:
     # Run any module initialization code.
     import module_init
     module_init.init()
   except ImportError:
     pass
 
-  task = sys.argv[1]
-
   task_module_name = f'clusterfuzz._internal.cron.{task}'
   with monitor.wrap_with_monitoring(), ndb_init.context():
     task_module = importlib.import_module(task_module_name)