Merge pull request #4009 from alexzurbonsen/fix-regression-false-positive-detection

Fix false positive detection heuristics

Author: AyanSinhaMahapatra

src/licensedcode/detection.py

@@ -1164,26 +1164,22 @@ def is_false_positive(license_matches, package_license=False):
     # FIXME: actually run copyright detection here?
     copyright_words = ["copyright", "(c)"]
     has_copyrights = all(
-        True
-        for license_match in license_matches
-        if any(
-            True
+        any(
+            word in license_match.matched_text().lower()
             for word in copyright_words
-            if word in license_match.matched_text().lower()
-        ) 
+        )
+        for license_match in license_matches 
     )
     has_full_relevance = all(
-        True
+        license_match.rule.relevance == 100
         for license_match in license_matches
-        if license_match.rule.relevance == 100
     )
     if has_copyrights or has_full_relevance:
         return False
 
     has_low_relevance = all(
-        True
+        license_match.rule.relevance < 60
         for license_match in license_matches
-        if license_match.rule.relevance < 60
     )
 
     start_line_region = min(

tests/licensedcode/data/plugin_license/scan/ffmpeg-license.expected.json

@@ -82,47 +82,6 @@
         }
       ]
     },
-    {
-      "identifier": "gpl_2_0_and_lgpl_2_0_plus-e597690d-3d55-0948-8697-914f460389e8",
-      "license_expression": "gpl-2.0 AND lgpl-2.0-plus",
-      "license_expression_spdx": "GPL-2.0-only AND LGPL-2.0-or-later",
-      "detection_count": 1,
-      "detection_log": [],
-      "reference_matches": [
-        {
-          "license_expression": "gpl-2.0",
-          "license_expression_spdx": "GPL-2.0-only",
-          "from_file": "ffmpeg-LICENSE.md",
-          "start_line": 100,
-          "end_line": 100,
-          "matcher": "2-aho",
-          "score": 100.0,
-          "matched_length": 1,
-          "match_coverage": 100.0,
-          "rule_relevance": 100,
-          "rule_identifier": "gpl-2.0_bare_single_word.RULE",
-          "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/gpl-2.0_bare_single_word.RULE",
-          "matched_text": "are incompatible with the GPLv2 and v3. We do not know for certain if their",
-          "matched_text_diagnostics": "GPLv2"
-        },
-        {
-          "license_expression": "lgpl-2.0-plus",
-          "license_expression_spdx": "LGPL-2.0-or-later",
-          "from_file": "ffmpeg-LICENSE.md",
-          "start_line": 101,
-          "end_line": 101,
-          "matcher": "2-aho",
-          "score": 60.0,
-          "matched_length": 1,
-          "match_coverage": 100.0,
-          "rule_relevance": 60,
-          "rule_identifier": "lgpl_bare_single_word.RULE",
-          "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/lgpl_bare_single_word.RULE",
-          "matched_text": "licenses are compatible with the LGPL.",
-          "matched_text_diagnostics": "LGPL."
-        }
-      ]
-    },
     {
       "identifier": "ijg_and_mit-e50613dc-8a09-65cc-c498-5d9527795382",
       "license_expression": "ijg AND mit",
@@ -182,31 +141,6 @@
         }
       ]
     },
-    {
-      "identifier": "lgpl_2_0_plus-e66c5b24-33ce-ea1b-0e3b-28952361ec32",
-      "license_expression": "lgpl-2.0-plus",
-      "license_expression_spdx": "LGPL-2.0-or-later",
-      "detection_count": 1,
-      "detection_log": [],
-      "reference_matches": [
-        {
-          "license_expression": "lgpl-2.0-plus",
-          "license_expression_spdx": "LGPL-2.0-or-later",
-          "from_file": "ffmpeg-LICENSE.md",
-          "start_line": 104,
-          "end_line": 104,
-          "matcher": "2-aho",
-          "score": 60.0,
-          "matched_length": 1,
-          "match_coverage": 100.0,
-          "rule_relevance": 60,
-          "rule_identifier": "lgpl_bare_single_word.RULE",
-          "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/lgpl_bare_single_word.RULE",
-          "matched_text": "be under a complex license mix that is more restrictive than the LGPL and that",
-          "matched_text_diagnostics": "LGPL"
-        }
-      ]
-    },
     {
       "identifier": "lgpl_2_1_plus_and_other_permissive_and_gpl_2_0_plus-666058ef-8c38-3b17-d8e7-448b304de833",
       "license_expression": "lgpl-2.1-plus AND other-permissive AND gpl-2.0-plus",
@@ -294,8 +228,8 @@
     {
       "path": "ffmpeg-LICENSE.md",
       "type": "file",
-      "detected_license_expression": "(lgpl-2.1-plus AND other-permissive AND gpl-2.0-plus) AND (lgpl-3.0 AND lgpl-3.0-plus AND (lgpl-3.0 AND gpl-3.0)) AND (ijg AND mit) AND gpl-1.0-plus AND (gpl-2.0 AND apache-2.0 AND lgpl-3.0-plus) AND (gpl-2.0 AND lgpl-2.0-plus) AND lgpl-2.0-plus",
-      "detected_license_expression_spdx": "(LGPL-2.1-or-later AND LicenseRef-scancode-other-permissive AND GPL-2.0-or-later) AND (LGPL-3.0-only AND LGPL-3.0-or-later AND (LGPL-3.0-only AND GPL-3.0-only)) AND (IJG AND MIT) AND GPL-1.0-or-later AND (GPL-2.0-only AND Apache-2.0 AND LGPL-3.0-or-later) AND (GPL-2.0-only AND LGPL-2.0-or-later) AND LGPL-2.0-or-later",
+      "detected_license_expression": "(lgpl-2.1-plus AND other-permissive AND gpl-2.0-plus) AND (lgpl-3.0 AND lgpl-3.0-plus AND (lgpl-3.0 AND gpl-3.0)) AND (ijg AND mit) AND gpl-1.0-plus AND (gpl-2.0 AND apache-2.0 AND lgpl-3.0-plus)",
+      "detected_license_expression_spdx": "(LGPL-2.1-or-later AND LicenseRef-scancode-other-permissive AND GPL-2.0-or-later) AND (LGPL-3.0-only AND LGPL-3.0-or-later AND (LGPL-3.0-only AND GPL-3.0-only)) AND (IJG AND MIT) AND GPL-1.0-or-later AND (GPL-2.0-only AND Apache-2.0 AND LGPL-3.0-or-later)",
       "license_detections": [
         {
           "license_expression": "lgpl-2.1-plus AND other-permissive AND gpl-2.0-plus",
@@ -514,73 +448,41 @@
           ],
           "detection_log": [],
           "identifier": "gpl_2_0_and_apache_2_0_and_lgpl_3_0_plus-c2393e5a-e531-304f-58a9-a6431d46d214"
-        },
+        }
+      ],
+      "license_clues": [
         {
-          "license_expression": "gpl-2.0 AND lgpl-2.0-plus",
-          "license_expression_spdx": "GPL-2.0-only AND LGPL-2.0-or-later",
-          "matches": [
-            {
-              "license_expression": "gpl-2.0",
-              "license_expression_spdx": "GPL-2.0-only",
-              "from_file": "ffmpeg-LICENSE.md",
-              "start_line": 100,
-              "end_line": 100,
-              "matcher": "2-aho",
-              "score": 100.0,
-              "matched_length": 1,
-              "match_coverage": 100.0,
-              "rule_relevance": 100,
-              "rule_identifier": "gpl-2.0_bare_single_word.RULE",
-              "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/gpl-2.0_bare_single_word.RULE",
-              "matched_text": "are incompatible with the GPLv2 and v3. We do not know for certain if their",
-              "matched_text_diagnostics": "GPLv2"
-            },
-            {
-              "license_expression": "lgpl-2.0-plus",
-              "license_expression_spdx": "LGPL-2.0-or-later",
-              "from_file": "ffmpeg-LICENSE.md",
-              "start_line": 101,
-              "end_line": 101,
-              "matcher": "2-aho",
-              "score": 60.0,
-              "matched_length": 1,
-              "match_coverage": 100.0,
-              "rule_relevance": 60,
-              "rule_identifier": "lgpl_bare_single_word.RULE",
-              "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/lgpl_bare_single_word.RULE",
-              "matched_text": "licenses are compatible with the LGPL.",
-              "matched_text_diagnostics": "LGPL."
-            }
-          ],
-          "detection_log": [],
-          "identifier": "gpl_2_0_and_lgpl_2_0_plus-e597690d-3d55-0948-8697-914f460389e8"
+          "license_expression": "gpl-2.0",
+          "license_expression_spdx": "GPL-2.0-only",
+          "from_file": "ffmpeg-LICENSE.md",
+          "start_line": 100,
+          "end_line": 100,
+          "matcher": "2-aho",
+          "score": 100.0,
+          "matched_length": 1,
+          "match_coverage": 100.0,
+          "rule_relevance": 100,
+          "rule_identifier": "gpl-2.0_bare_single_word.RULE",
+          "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/gpl-2.0_bare_single_word.RULE",
+          "matched_text": "are incompatible with the GPLv2 and v3. We do not know for certain if their",
+          "matched_text_diagnostics": "GPLv2"
         },
         {
           "license_expression": "lgpl-2.0-plus",
           "license_expression_spdx": "LGPL-2.0-or-later",
-          "matches": [
-            {
-              "license_expression": "lgpl-2.0-plus",
-              "license_expression_spdx": "LGPL-2.0-or-later",
-              "from_file": "ffmpeg-LICENSE.md",
-              "start_line": 104,
-              "end_line": 104,
-              "matcher": "2-aho",
-              "score": 60.0,
-              "matched_length": 1,
-              "match_coverage": 100.0,
-              "rule_relevance": 60,
-              "rule_identifier": "lgpl_bare_single_word.RULE",
-              "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/lgpl_bare_single_word.RULE",
-              "matched_text": "be under a complex license mix that is more restrictive than the LGPL and that",
-              "matched_text_diagnostics": "LGPL"
-            }
-          ],
-          "detection_log": [],
-          "identifier": "lgpl_2_0_plus-e66c5b24-33ce-ea1b-0e3b-28952361ec32"
-        }
-      ],
-      "license_clues": [
+          "from_file": "ffmpeg-LICENSE.md",
+          "start_line": 101,
+          "end_line": 101,
+          "matcher": "2-aho",
+          "score": 60.0,
+          "matched_length": 1,
+          "match_coverage": 100.0,
+          "rule_relevance": 60,
+          "rule_identifier": "lgpl_bare_single_word.RULE",
+          "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/lgpl_bare_single_word.RULE",
+          "matched_text": "licenses are compatible with the LGPL.",
+          "matched_text_diagnostics": "LGPL."
+        },
         {
           "license_expression": "proprietary-license",
           "license_expression_spdx": "LicenseRef-scancode-proprietary-license",
@@ -596,10 +498,26 @@
           "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/proprietary-license_490.RULE",
           "matched_text": "If you wish to enable these libraries, pass --enable-nonfree to configure.",
           "matched_text_diagnostics": "enable-nonfree"
+        },
+        {
+          "license_expression": "lgpl-2.0-plus",
+          "license_expression_spdx": "LGPL-2.0-or-later",
+          "from_file": "ffmpeg-LICENSE.md",
+          "start_line": 104,
+          "end_line": 104,
+          "matcher": "2-aho",
+          "score": 60.0,
+          "matched_length": 1,
+          "match_coverage": 100.0,
+          "rule_relevance": 60,
+          "rule_identifier": "lgpl_bare_single_word.RULE",
+          "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/lgpl_bare_single_word.RULE",
+          "matched_text": "be under a complex license mix that is more restrictive than the LGPL and that",
+          "matched_text_diagnostics": "LGPL"
         }
       ],
       "percentage_of_license_text": 34.78,
       "scan_errors": []
     }
   ]
-}
+}