Extract conscrypt native libs from Snowflake and Google_Api (#13211)

* Remove expected constrypt native libs from both Google_Api and Snowflake

* Extract conscrypt native libs within Google_Api

* Extract conscrypt native libs within Snowflake

* fmt

Author: Akirathan

build.sbt

@@ -5271,7 +5271,7 @@ lazy val `std-google-api` = project
           unmanagedClasspath = (Compile / unmanagedJars).value,
           previousRun        = prev
         )
-      StdBits
+      val grpc = StdBits
         .extractNativeLibsFromGrpc(
           `google-api-polyglot-root`,
           `google-api-native-libs`,
@@ -5283,6 +5283,18 @@ lazy val `std-google-api` = project
           cacheStoreFactory  = cacheStoreFactory,
           previousRun        = prev
         )
+      val conscrypt = StdBits
+        .extractNativeLibsFromConscrypt(
+          `google-api-polyglot-root`,
+          `google-api-native-libs`,
+          updateReport       = (Compile / update).value,
+          logger             = streams.value.log,
+          moduleName         = moduleName.value,
+          scalaBinaryVersion = scalaBinaryVersion.value,
+          cacheStoreFactory  = cacheStoreFactory,
+          previousRun        = prev
+        )
+      grpc.appended(conscrypt)
     }.value,
     cleanPolyglotRoot := Def.task {
       import sbt.util.CacheImplicits._
@@ -5462,7 +5474,7 @@ lazy val `std-snowflake` = project
           unmanagedClasspath                = (Compile / unmanagedJars).value,
           previousRun                       = prev
         )
-      StdBits
+      val grpc = StdBits
         .extractNativeLibsFromGrpc(
           `std-snowflake-polyglot-root`,
           `std-snowflake-native-libs`,
@@ -5474,6 +5486,18 @@ lazy val `std-snowflake` = project
           cacheStoreFactory  = cacheStoreFactory,
           previousRun        = prev
         )
+      val conscrypt = StdBits
+        .extractNativeLibsFromConscrypt(
+          `std-snowflake-polyglot-root`,
+          `std-snowflake-native-libs`,
+          updateReport       = (Compile / update).value,
+          logger             = streams.value.log,
+          moduleName         = moduleName.value,
+          scalaBinaryVersion = scalaBinaryVersion.value,
+          cacheStoreFactory  = cacheStoreFactory,
+          previousRun        = prev
+        )
+      grpc.appended(conscrypt)
     }.value,
     cleanPolyglotRoot := Def.task {
       import sbt.util.CacheImplicits._

project/AnalysisOfExtractedNativeLibs.scala

@@ -14,6 +14,12 @@ case class AnalysisOfExtractedNativeLibs(
     libs.find(_.from == srcJar)
 
   def isOutdated: Boolean = libs.exists(_.isOutdated)
+
+  def appended(
+    other: AnalysisOfExtractedNativeLibs
+  ): AnalysisOfExtractedNativeLibs = {
+    AnalysisOfExtractedNativeLibs(libs ++ other.libs)
+  }
 }
 object AnalysisOfExtractedNativeLibs {
   import sjsonnew.{:*:, LList, LNil}

project/StdBits.scala

@@ -506,6 +506,99 @@ object StdBits {
     )
   }
 
+  /** Extract native libraries from `org.conscrypt:conscrypt-openjdk-uber:2.5.2` jar, which is
+    * a transitive dependency of
+    * `com.google.analytics:google-analytics-admin:0.66.0` and of
+    * `net.snowflake:snowflake-jdbc-thin:3.15.0`.
+    *
+    * Currently, it is included in both `Standard.Google_Api` and `Standard.Snowflake` libraries.
+    *
+    * Names of the native libraries in jar:
+    * - `META-INF/native/conscrypt_openjdk_jni-windows-x86.dll`
+    * - `META-INF/native/conscrypt_openjdk_jni-windows-x86_64.dll`
+    * - `META-INF/native/libconscrypt_openjdk_jni-linux-x86_64.so`
+    * - `META-INF/native/libconscrypt_openjdk_jni-osx-x86_64.dylib`
+    *
+    * The jar is signed, so we also have to remove `META-INF/SIGNING.SF`.
+    */
+  def extractNativeLibsFromConscrypt(
+    polyglotRootDir: File,
+    nativeLibsDir: File,
+    updateReport: UpdateReport,
+    logger: ManagedLogger,
+    moduleName: String,
+    scalaBinaryVersion: String,
+    cacheStoreFactory: CacheStoreFactory,
+    previousRun: Option[AnalysisOfExtractedNativeLibs]
+  ): AnalysisOfExtractedNativeLibs = {
+    if (previousRun.exists(!_.isOutdated)) {
+      return previousRun.get
+    }
+    val osName     = plainOsName().replace("macos", "osx")
+    val validOsExt = osExt()
+    val validArch  = arch().replace("-", "_")
+    val prefix     = "META-INF/native"
+    val entriesToRemove = Seq(
+      "META-INF/SIGNINGC.SF",
+      "META-INF/SIGNINGC.RSA"
+    )
+    val conscryptVersion = "2.5.2"
+
+    def renameFunc(entryName: String): Option[String] = {
+      val strippedEntryName = entryName.substring(prefix.length + 1)
+      val pattern           = "^(.+)-(\\w+)-([\\w_]+)(\\.\\w+)$".r
+      strippedEntryName match {
+        case pattern(libname, entryOs, entryArch, entryExt) =>
+          if (
+            !entryOs.equals(osName) ||
+            !entryArch.equals(validArch) ||
+            !entryExt.equals(validOsExt)
+          ) {
+            None
+          } else {
+            val outputArch = validArch.replace("x86_64", "amd64")
+            Some(s"$outputArch/$osName/$libname$entryExt")
+          }
+        case _ =>
+          throw new RuntimeException(
+            s"Unexpected entry name format: $strippedEntryName"
+          )
+      }
+    }
+
+    val conscryptJar = JPMSUtils
+      .filterModulesFromUpdate(
+        updateReport,
+        Seq("org.conscrypt" % "conscrypt-openjdk-uber" % conscryptVersion),
+        logger,
+        moduleName,
+        scalaBinaryVersion,
+        shouldContainAll = true
+      )
+      .head
+    val outputJar =
+      (polyglotRootDir / s"conscrypt-openjdk-uber-$conscryptVersion.jar").toPath
+    val extractedLibs = JARUtils.extractFilesFromJar(
+      conscryptJar.toPath,
+      Some(prefix),
+      Some(outputJar),
+      nativeLibsDir.toPath,
+      renameFunc,
+      logger,
+      cacheStoreFactory,
+      previousRun.flatMap(_.forJar(conscryptJar))
+    )
+    JARUtils.removeEntriesFromJar(
+      outputJar,
+      entryName => entriesToRemove.contains(entryName)
+    )
+    AnalysisOfExtractedNativeLibs(
+      conscryptJar,
+      extractedLibs.getOrElse(Nil),
+      Some(outputJar.toFile)
+    )
+  }
+
   def ensureDirExistsAndIsClean(
     path: Path,
     logger: sbt.util.Logger,

test/Base_Tests/data/native_libs.json

@@ -125,18 +125,6 @@
     "META-INF/resources/nfi-native/libnfi/darwin/aarch64/bin/libtrufflenfi.dylib",
     "META-INF/resources/nfi-native/libnfi/windows/amd64/bin/trufflenfi.dll"
   ],
-  "lib/Standard/Snowflake/0.0.0-dev/polyglot/java/conscrypt-openjdk-uber-2.5.2.jar": [
-    "META-INF/native/conscrypt_openjdk_jni-windows-x86.dll",
-    "META-INF/native/conscrypt_openjdk_jni-windows-x86_64.dll",
-    "META-INF/native/libconscrypt_openjdk_jni-linux-x86_64.so",
-    "META-INF/native/libconscrypt_openjdk_jni-osx-x86_64.dylib"
-  ],
-  "lib/Standard/Google_Api/0.0.0-dev/polyglot/java/conscrypt-openjdk-uber-2.5.2.jar": [
-    "META-INF/native/conscrypt_openjdk_jni-windows-x86.dll",
-    "META-INF/native/conscrypt_openjdk_jni-windows-x86_64.dll",
-    "META-INF/native/libconscrypt_openjdk_jni-linux-x86_64.so",
-    "META-INF/native/libconscrypt_openjdk_jni-osx-x86_64.dylib"
-  ],
   "lib/Standard/Microsoft/0.0.0-dev/polyglot/java/jna-wrapper-assembly-0.1.0-SNAPSHOT.jar": [
     "com/sun/jna/freebsd-x86-64/libjnidispatch.so",
     "com/sun/jna/freebsd-x86/libjnidispatch.so",