feat(scanner): Merge duplicate scan results that share a provenance

When the SpdxDocumentFile package manager is used, the *project* and all
contained *packages* often resolve to the **same VCS provenance** (e.g. the
root of the Git repository).
Before this change ORT stored two separate `ScanResult`s for such a
provenance – one keyed to the project, one keyed to the package.

That caused two follow-on problems:

* Both results appeared in the `OrtResult`, so evaluators saw **duplicate
  findings** for the *same* source tree.
* Because projects and packages are handled by different rules the package
  result was additionally **padded with a `SpdxConstants.NONE` finding**
  whenever `includeFilesWithoutFindings` was enabled.
  The evaluator therefore compared *real* license findings from the project
  result with `NONE` from the package result and failed with a violation.

This patch

* groups scan results by the pair `(provenance, scanner)` and folds them
  into a single `ScanResult`,
* unions the inner finding sets to avoid duplicates, and
* performs the "pad with NONE" step only **after** deduplication, so every
  path is represented exactly once.

As a consequence the evaluator now receives one consistent set of license
findings per provenance / scanner, eliminating the false mismatch.

Signed-off-by: Jonatan Männchen <jonatan@maennchen.ch>

Author: maennchen

scanner/src/funTest/kotlin/scanners/ScannerIntegrationFunTest.kt

@@ -34,6 +34,7 @@ import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.PackageReference
 import org.ossreviewtoolkit.model.PackageType
 import org.ossreviewtoolkit.model.Project
+import org.ossreviewtoolkit.model.Repository
 import org.ossreviewtoolkit.model.ScanSummary
 import org.ossreviewtoolkit.model.Scope
 import org.ossreviewtoolkit.model.TextLocation
@@ -61,7 +62,7 @@ import org.ossreviewtoolkit.utils.test.readResource
 
 class ScannerIntegrationFunTest : WordSpec({
     "Scanning all packages corresponding to a single VCS" should {
-        val analyzerResult = createAnalyzerResult(pkg0, pkg1, pkg2, pkg3, pkg4)
+        val analyzerResult = createAnalyzerResult(project0, pkg0, pkg1, pkg2, pkg3, pkg4)
         val ortResult = createScanner().scan(analyzerResult, skipExcluded = false, emptyMap())
 
         "return the expected ORT result" {
@@ -91,7 +92,7 @@ class ScannerIntegrationFunTest : WordSpec({
 
     "Scanning a subset of the packages corresponding to a single VCS" should {
         "return the expected ORT result" {
-            val analyzerResult = createAnalyzerResult(pkg1, pkg3)
+            val analyzerResult = createAnalyzerResult(project0, pkg1, pkg3)
             val expectedResult = readResource("/scanner-integration-subset-pkgs-expected-ort-result.yml")
 
             val ortResult = createScanner().scan(analyzerResult, skipExcluded = false, emptyMap())
@@ -127,26 +128,31 @@ internal fun createScanner(scannerWrappers: Map<PackageType, List<ScannerWrapper
     )
 }
 
-private fun createAnalyzerResult(vararg packages: Package): OrtResult {
+private fun createAnalyzerResult(project: Project, vararg packages: Package): OrtResult {
     val scope = Scope(
         name = "deps",
         dependencies = packages.mapTo(mutableSetOf()) { PackageReference(it.id) }
     )
 
-    val project = Project.EMPTY.copy(
-        id = createId("project"),
+    val projectWithScope = project.copy(
         scopeDependencies = setOf(scope)
     )
 
     val analyzerRun = AnalyzerRun.EMPTY.copy(
         result = AnalyzerResult.EMPTY.copy(
-            projects = setOf(project),
+            projects = setOf(projectWithScope),
             packages = packages.toSet()
         ),
         config = AnalyzerConfiguration(enabledPackageManagers = emptyList())
     )
 
-    return OrtResult.EMPTY.copy(analyzer = analyzerRun)
+    return OrtResult.EMPTY.copy(
+        analyzer = analyzerRun,
+        repository = Repository.EMPTY.copy(
+            vcsProcessed = projectWithScope.vcsProcessed,
+            vcs = projectWithScope.vcs
+        )
+    )
 }
 
 private fun createId(name: String): Identifier = Identifier("Dummy::$name:1.0.0")
@@ -158,6 +164,24 @@ private fun createPackage(name: String, vcs: VcsInfo): Package =
         vcsProcessed = vcs.normalize()
     )
 
+private fun createProject(name: String, vcs: VcsInfo): Project =
+    Project.EMPTY.copy(
+        id = createId(name),
+        vcs = vcs,
+        vcsProcessed = vcs.normalize()
+    )
+
+// Project package
+private val project0 = createProject(
+    name = "project",
+    vcs = VcsInfo(
+        type = VcsType.GIT,
+        url = "https://github.com/oss-review-toolkit/ort-test-data-scanner.git",
+        revision = "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec",
+        path = ""
+    )
+)
+
 // A package with an empty VCS path.
 private val pkg0 = createPackage(
     name = "pkg0",

scanner/src/funTest/resources/scanner-integration-all-pkgs-expected-ort-result.yml

@@ -1,14 +1,14 @@
 ---
 repository:
   vcs:
-    type: ""
-    url: ""
-    revision: ""
+    type: "Git"
+    url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+    revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
     path: ""
   vcs_processed:
-    type: ""
-    url: ""
-    revision: ""
+    type: "Git"
+    url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+    revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
     path: ""
   config: {}
 analyzer:
@@ -34,14 +34,14 @@ analyzer:
       declared_licenses: []
       declared_licenses_processed: {}
       vcs:
-        type: ""
-        url: ""
-        revision: ""
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+        revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
         path: ""
       vcs_processed:
-        type: ""
-        url: ""
-        revision: ""
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+        revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
         path: ""
       homepage_url: ""
       scopes:
@@ -289,12 +289,24 @@ scanner:
         revision: "6431fd85188db22b942deb66c7a8c1a53921fc35"
         path: ""
   - id: "Dummy::project:1.0.0"
-    package_provenance_resolution_issue:
-      timestamp: "1970-01-01T00:00:00Z"
-      source: "Scanner"
-      message: "IOException: Could not resolve provenance for package 'Dummy::project:1.0.0'\
-        \ for source code origins [VCS, ARTIFACT]."
-      severity: "ERROR"
+    package_provenance:
+      vcs_info:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+        revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+        path: ""
+      resolved_revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+    sub_repositories:
+      pkg3/subrepo:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner-subrepo.git"
+        revision: "a732695e03efcbd74539208af98c297ee86e49d5"
+        path: ""
+      pkg4/subrepo:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner-subrepo2.git"
+        revision: "6431fd85188db22b942deb66c7a8c1a53921fc35"
+        path: ""
   scan_results:
   - provenance:
       vcs_info:

scanner/src/funTest/resources/scanner-integration-expected-file-lists.yml

@@ -90,3 +90,38 @@ Dummy::pkg4:1.0.0:
     sha1: "ae8044f7fce7ee914a853c30c3085895e9be8b9c"
   - path: "pkg4/subrepo/pkg-s2/pkg-s2.txt"
     sha1: "37996d13eceb6b29db43a381ce8df375b5eee8e9"
+Dummy::project:1.0.0:
+  provenance:
+    vcs_info:
+      type: "Git"
+      url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+      revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+      path: ""
+    resolved_revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+  files:
+  - path: ".gitmodules"
+    sha1: "d7f070ddbe0b6dd8a173714d565a1240dd96eacd"
+  - path: "LICENSE"
+    sha1: "7df059597099bb7dcf25d2a9aedfaf4465f72d8d"
+  - path: "README"
+    sha1: "82cfc115138054ce5b5e6839f38687c9d7186710"
+  - path: "pkg1/pkg1.txt"
+    sha1: "22eb73bd30d47540a4e05781f0f6e07640857cae"
+  - path: "pkg2/pkg2.txt"
+    sha1: "cc8f97cebe1dc0ed889a31f504bcf491d5241aaa"
+  - path: "pkg3/pkg3.txt"
+    sha1: "859d66be2d153968cdaa8ec7265270c241eea024"
+  - path: "pkg3/subrepo/LICENSE"
+    sha1: "7df059597099bb7dcf25d2a9aedfaf4465f72d8d"
+  - path: "pkg3/subrepo/README"
+    sha1: "ae8044f7fce7ee914a853c30c3085895e9be8b9c"
+  - path: "pkg3/subrepo/pkg-s1/pkg-s1.txt"
+    sha1: "e5fb17f8f4f4ef0748bb5ba137fd0e091dd5a1f6"
+  - path: "pkg4/pkg4.txt"
+    sha1: "3cba29011be2b9d59f6204d6fa0a386b1b2dbd90"
+  - path: "pkg4/subrepo/LICENSE"
+    sha1: "7df059597099bb7dcf25d2a9aedfaf4465f72d8d"
+  - path: "pkg4/subrepo/README"
+    sha1: "ae8044f7fce7ee914a853c30c3085895e9be8b9c"
+  - path: "pkg4/subrepo/pkg-s2/pkg-s2.txt"
+    sha1: "37996d13eceb6b29db43a381ce8df375b5eee8e9"

scanner/src/funTest/resources/scanner-integration-expected-scan-results.yml

@@ -215,17 +215,83 @@ Dummy::pkg4:1.0.0:
         start_line: -1
         end_line: -1
 Dummy::project:1.0.0:
-- provenance: {}
+- provenance:
+    vcs_info:
+      type: "Git"
+      url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+      revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+      path: ""
+    resolved_revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
   scanner:
-    name: "ProvenanceResolver"
-    version: ""
+    name: "Dummy"
+    version: "1.0.0"
     configuration: ""
   summary:
     start_time: "1970-01-01T00:00:00Z"
     end_time: "1970-01-01T00:00:00Z"
-    issues:
-    - timestamp: "1970-01-01T00:00:00Z"
-      source: "Scanner"
-      message: "IOException: Could not resolve provenance for package 'Dummy::project:1.0.0'\
-        \ for source code origins [VCS, ARTIFACT]."
-      severity: "ERROR"
+    licenses:
+    - license: "NOASSERTION"
+      location:
+        path: ".gitmodules"
+        start_line: -1
+        end_line: -1
+    - license: "NOASSERTION"
+      location:
+        path: "LICENSE"
+        start_line: -1
+        end_line: -1
+    - license: "NOASSERTION"
+      location:
+        path: "README"
+        start_line: -1
+        end_line: -1
+    - license: "NOASSERTION"
+      location:
+        path: "pkg1/pkg1.txt"
+        start_line: -1
+        end_line: -1
+    - license: "NOASSERTION"
+      location:
+        path: "pkg2/pkg2.txt"
+        start_line: -1
+        end_line: -1
+    - license: "NOASSERTION"
+      location:
+        path: "pkg3/pkg3.txt"
+        start_line: -1
+        end_line: -1
+    - license: "NOASSERTION"
+      location:
+        path: "pkg3/subrepo/LICENSE"
+        start_line: -1
+        end_line: -1
+    - license: "NOASSERTION"
+      location:
+        path: "pkg3/subrepo/README"
+        start_line: -1
+        end_line: -1
+    - license: "NOASSERTION"
+      location:
+        path: "pkg3/subrepo/pkg-s1/pkg-s1.txt"
+        start_line: -1
+        end_line: -1
+    - license: "NOASSERTION"
+      location:
+        path: "pkg4/pkg4.txt"
+        start_line: -1
+        end_line: -1
+    - license: "NOASSERTION"
+      location:
+        path: "pkg4/subrepo/LICENSE"
+        start_line: -1
+        end_line: -1
+    - license: "NOASSERTION"
+      location:
+        path: "pkg4/subrepo/README"
+        start_line: -1
+        end_line: -1
+    - license: "NOASSERTION"
+      location:
+        path: "pkg4/subrepo/pkg-s2/pkg-s2.txt"
+        start_line: -1
+        end_line: -1