feat(scanner): Merge duplicate scan results that share a provenance

When the SpdxDocumentFile package manager is used, the *project* and all
contained *packages* often resolve to the **same VCS provenance** (e.g. the
root of the Git repository).
Before this change ORT stored two separate `ScanResult`s for such a
provenance – one keyed to the project, one keyed to the package.

That caused two follow-on problems:

* Both results appeared in the `OrtResult`, so evaluators saw **duplicate
  findings** for the *same* source tree.
* Because projects and packages are handled by different rules the package
  result was additionally **padded with a `SpdxConstants.NONE` finding**
  whenever `includeFilesWithoutFindings` was enabled.
  The evaluator therefore compared *real* license findings from the project
  result with `NONE` from the package result and failed with a violation.

This patch

* merges `ScannerRun`, and
* performs the "pad with NONE" step only **after** deduplication, so every
  path is represented exactly once.

As a consequence the evaluator now receives one consistent set of license
findings per provenance / scanner, eliminating the false mismatch.

Signed-off-by: Jonatan Männchen <jonatan@maennchen.ch>

Author: maennchen

scanner/src/funTest/kotlin/scanners/ScannerIntegrationFunTest.kt

@@ -20,6 +20,7 @@
 package org.ossreviewtoolkit.scanner.scanners
 
 import io.kotest.core.spec.style.WordSpec
+import io.kotest.matchers.collections.shouldHaveSize
 import io.kotest.matchers.shouldBe
 
 import java.io.File
@@ -34,6 +35,7 @@ import org.ossreviewtoolkit.model.Package
 import org.ossreviewtoolkit.model.PackageReference
 import org.ossreviewtoolkit.model.PackageType
 import org.ossreviewtoolkit.model.Project
+import org.ossreviewtoolkit.model.Repository
 import org.ossreviewtoolkit.model.ScanSummary
 import org.ossreviewtoolkit.model.Scope
 import org.ossreviewtoolkit.model.TextLocation
@@ -100,6 +102,15 @@ class ScannerIntegrationFunTest : WordSpec({
                 patchExpectedResult(expectedResult)
         }
     }
+
+    "Scanning a project with the same provenance as packages" should {
+        "not have duplicated scan results" {
+            val analyzerResult = createAnalyzerResultWithProject(project0, pkg0)
+            val ortResult = createScanner().scan(analyzerResult, skipExcluded = false, emptyMap())
+
+            ortResult.getScanResultsForId(project0.id) shouldHaveSize 1
+        }
+    }
 })
 
 internal fun createScanner(scannerWrappers: Map<PackageType, List<ScannerWrapper>>? = null): Scanner {
@@ -128,25 +139,38 @@ internal fun createScanner(scannerWrappers: Map<PackageType, List<ScannerWrapper
 }
 
 private fun createAnalyzerResult(vararg packages: Package): OrtResult {
+    val project = Project.EMPTY.copy(
+        id = createId("project")
+    )
+
+    return createAnalyzerResultWithProject(project, *packages)
+}
+
+private fun createAnalyzerResultWithProject(project: Project, vararg packages: Package): OrtResult {
     val scope = Scope(
         name = "deps",
         dependencies = packages.mapTo(mutableSetOf()) { PackageReference(it.id) }
     )
 
-    val project = Project.EMPTY.copy(
-        id = createId("project"),
+    val projectWithScope = project.copy(
         scopeDependencies = setOf(scope)
     )
 
     val analyzerRun = AnalyzerRun.EMPTY.copy(
         result = AnalyzerResult.EMPTY.copy(
-            projects = setOf(project),
+            projects = setOf(projectWithScope),
             packages = packages.toSet()
         ),
         config = AnalyzerConfiguration(enabledPackageManagers = emptyList())
     )
 
-    return OrtResult.EMPTY.copy(analyzer = analyzerRun)
+    return OrtResult.EMPTY.copy(
+        analyzer = analyzerRun,
+        repository = Repository.EMPTY.copy(
+            vcsProcessed = projectWithScope.vcsProcessed,
+            vcs = projectWithScope.vcs
+        )
+    )
 }
 
 private fun createId(name: String): Identifier = Identifier("Dummy::$name:1.0.0")
@@ -158,6 +182,23 @@ private fun createPackage(name: String, vcs: VcsInfo): Package =
         vcsProcessed = vcs.normalize()
     )
 
+private fun createProject(name: String, vcs: VcsInfo): Project =
+    Project.EMPTY.copy(
+        id = createId(name),
+        vcs = vcs,
+        vcsProcessed = vcs.normalize()
+    )
+
+private val project0 = createProject(
+    name = "project",
+    vcs = VcsInfo(
+        type = VcsType.GIT,
+        url = "https://github.com/oss-review-toolkit/ort-test-data-scanner.git",
+        revision = "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec",
+        path = ""
+    )
+)
+
 // A package with an empty VCS path.
 private val pkg0 = createPackage(
     name = "pkg0",

scanner/src/main/kotlin/Scanner.kt

@@ -107,8 +107,6 @@ class Scanner(
     )
 
     suspend fun scan(ortResult: OrtResult, skipExcluded: Boolean, labels: Map<String, String>): OrtResult {
-        val startTime = Instant.now()
-
         val projectPackages = ortResult.getProjects(skipExcluded).mapTo(mutableSetOf()) { it.toPackage() }
         val projectResults = scan(
             projectPackages,
@@ -134,8 +132,6 @@ class Scanner(
             )
         )
 
-        val endTime = Instant.now()
-
         val toolVersions = mutableMapOf<String, String>()
 
         scannerWrappers.values.flatten().forEach { scanner ->
@@ -144,19 +140,14 @@ class Scanner(
             }
         }
 
-        val scannerRun = ScannerRun(
-            startTime = startTime,
-            endTime = endTime,
-            environment = Environment(toolVersions = toolVersions),
-            config = scannerConfig,
-            provenances = projectResults.provenances + packageResults.provenances,
-            scanResults = projectResults.scanResults + packageResults.scanResults,
-            issues = projectResults.issues + packageResults.issues,
-            files = projectResults.files + packageResults.files,
-            scanners = projectResults.scanners + packageResults.scanners
+        val scannerRun = (projectResults + packageResults).copy(
+            environment = Environment(toolVersions = toolVersions)
         )
 
-        return ortResult.copy(scanner = scannerRun)
+        val paddedScannerRun = scannerRun.takeUnless { scannerConfig.includeFilesWithoutFindings }
+            ?: scannerRun.padNoneLicenseFindings()
+
+        return ortResult.copy(scanner = paddedScannerRun)
     }
 
     internal suspend fun scan(packages: Set<Package>, context: ScanContext): ScannerRun {
@@ -170,6 +161,8 @@ class Scanner(
 
         val controller = ScanController(packages, scannerWrappers, scannerConfig)
 
+        val startTime = Instant.now()
+
         resolvePackageProvenances(controller)
         resolveNestedProvenances(controller)
 
@@ -182,6 +175,8 @@ class Scanner(
         createFileLists(controller)
         createMissingArchives(controller)
 
+        val endTime = Instant.now()
+
         val provenances = packages.mapTo(mutableSetOf()) { pkg ->
             val packageProvenance = controller.getPackageProvenance(pkg.id)
 
@@ -216,17 +211,17 @@ class Scanner(
 
         val issues = controller.getIssues()
 
-        val scannerRun = ScannerRun.EMPTY.copy(
+        return ScannerRun(
+            startTime = startTime,
+            endTime = endTime,
+            environment = Environment(),
             config = scannerConfig,
             provenances = provenances,
             scanResults = scanResults,
             files = files,
             scanners = scanners,
             issues = issues
         )
-
-        return scannerRun.takeUnless { scannerConfig.includeFilesWithoutFindings }
-            ?: scannerRun.padNoneLicenseFindings()
     }
 
     private suspend fun resolvePackageProvenances(controller: ScanController) {