diff --git a/build.sbt b/build.sbt index 0cf7d659d0c6..012f62d629d6 100644 --- a/build.sbt +++ b/build.sbt @@ -5271,7 +5271,7 @@ lazy val `std-google-api` = project unmanagedClasspath = (Compile / unmanagedJars).value, previousRun = prev ) - StdBits + val grpc = StdBits .extractNativeLibsFromGrpc( `google-api-polyglot-root`, `google-api-native-libs`, @@ -5283,6 +5283,18 @@ lazy val `std-google-api` = project cacheStoreFactory = cacheStoreFactory, previousRun = prev ) + val conscrypt = StdBits + .extractNativeLibsFromConscrypt( + `google-api-polyglot-root`, + `google-api-native-libs`, + updateReport = (Compile / update).value, + logger = streams.value.log, + moduleName = moduleName.value, + scalaBinaryVersion = scalaBinaryVersion.value, + cacheStoreFactory = cacheStoreFactory, + previousRun = prev + ) + grpc.appended(conscrypt) }.value, cleanPolyglotRoot := Def.task { import sbt.util.CacheImplicits._ @@ -5462,7 +5474,7 @@ lazy val `std-snowflake` = project unmanagedClasspath = (Compile / unmanagedJars).value, previousRun = prev ) - StdBits + val grpc = StdBits .extractNativeLibsFromGrpc( `std-snowflake-polyglot-root`, `std-snowflake-native-libs`, @@ -5474,6 +5486,18 @@ lazy val `std-snowflake` = project cacheStoreFactory = cacheStoreFactory, previousRun = prev ) + val conscrypt = StdBits + .extractNativeLibsFromConscrypt( + `std-snowflake-polyglot-root`, + `std-snowflake-native-libs`, + updateReport = (Compile / update).value, + logger = streams.value.log, + moduleName = moduleName.value, + scalaBinaryVersion = scalaBinaryVersion.value, + cacheStoreFactory = cacheStoreFactory, + previousRun = prev + ) + grpc.appended(conscrypt) }.value, cleanPolyglotRoot := Def.task { import sbt.util.CacheImplicits._ diff --git a/project/AnalysisOfExtractedNativeLibs.scala b/project/AnalysisOfExtractedNativeLibs.scala index 93bc4e58f2e4..50aad4eb1d0d 100644 --- a/project/AnalysisOfExtractedNativeLibs.scala +++ b/project/AnalysisOfExtractedNativeLibs.scala @@ -14,6 +14,12 @@ case class AnalysisOfExtractedNativeLibs( libs.find(_.from == srcJar) def isOutdated: Boolean = libs.exists(_.isOutdated) + + def appended( + other: AnalysisOfExtractedNativeLibs + ): AnalysisOfExtractedNativeLibs = { + AnalysisOfExtractedNativeLibs(libs ++ other.libs) + } } object AnalysisOfExtractedNativeLibs { import sjsonnew.{:*:, LList, LNil} diff --git a/project/StdBits.scala b/project/StdBits.scala index e29ecfdfacc6..9f42f314cd1a 100644 --- a/project/StdBits.scala +++ b/project/StdBits.scala @@ -506,6 +506,99 @@ object StdBits { ) } + /** Extract native libraries from `org.conscrypt:conscrypt-openjdk-uber:2.5.2` jar, which is + * a transitive dependency of + * `com.google.analytics:google-analytics-admin:0.66.0` and of + * `net.snowflake:snowflake-jdbc-thin:3.15.0`. + * + * Currently, it is included in both `Standard.Google_Api` and `Standard.Snowflake` libraries. + * + * Names of the native libraries in jar: + * - `META-INF/native/conscrypt_openjdk_jni-windows-x86.dll` + * - `META-INF/native/conscrypt_openjdk_jni-windows-x86_64.dll` + * - `META-INF/native/libconscrypt_openjdk_jni-linux-x86_64.so` + * - `META-INF/native/libconscrypt_openjdk_jni-osx-x86_64.dylib` + * + * The jar is signed, so we also have to remove `META-INF/SIGNING.SF`. + */ + def extractNativeLibsFromConscrypt( + polyglotRootDir: File, + nativeLibsDir: File, + updateReport: UpdateReport, + logger: ManagedLogger, + moduleName: String, + scalaBinaryVersion: String, + cacheStoreFactory: CacheStoreFactory, + previousRun: Option[AnalysisOfExtractedNativeLibs] + ): AnalysisOfExtractedNativeLibs = { + if (previousRun.exists(!_.isOutdated)) { + return previousRun.get + } + val osName = plainOsName().replace("macos", "osx") + val validOsExt = osExt() + val validArch = arch().replace("-", "_") + val prefix = "META-INF/native" + val entriesToRemove = Seq( + "META-INF/SIGNINGC.SF", + "META-INF/SIGNINGC.RSA" + ) + val conscryptVersion = "2.5.2" + + def renameFunc(entryName: String): Option[String] = { + val strippedEntryName = entryName.substring(prefix.length + 1) + val pattern = "^(.+)-(\\w+)-([\\w_]+)(\\.\\w+)$".r + strippedEntryName match { + case pattern(libname, entryOs, entryArch, entryExt) => + if ( + !entryOs.equals(osName) || + !entryArch.equals(validArch) || + !entryExt.equals(validOsExt) + ) { + None + } else { + val outputArch = validArch.replace("x86_64", "amd64") + Some(s"$outputArch/$osName/$libname$entryExt") + } + case _ => + throw new RuntimeException( + s"Unexpected entry name format: $strippedEntryName" + ) + } + } + + val conscryptJar = JPMSUtils + .filterModulesFromUpdate( + updateReport, + Seq("org.conscrypt" % "conscrypt-openjdk-uber" % conscryptVersion), + logger, + moduleName, + scalaBinaryVersion, + shouldContainAll = true + ) + .head + val outputJar = + (polyglotRootDir / s"conscrypt-openjdk-uber-$conscryptVersion.jar").toPath + val extractedLibs = JARUtils.extractFilesFromJar( + conscryptJar.toPath, + Some(prefix), + Some(outputJar), + nativeLibsDir.toPath, + renameFunc, + logger, + cacheStoreFactory, + previousRun.flatMap(_.forJar(conscryptJar)) + ) + JARUtils.removeEntriesFromJar( + outputJar, + entryName => entriesToRemove.contains(entryName) + ) + AnalysisOfExtractedNativeLibs( + conscryptJar, + extractedLibs.getOrElse(Nil), + Some(outputJar.toFile) + ) + } + def ensureDirExistsAndIsClean( path: Path, logger: sbt.util.Logger, diff --git a/test/Base_Tests/data/native_libs.json b/test/Base_Tests/data/native_libs.json index cccf0dd7ff36..061d8ac54394 100644 --- a/test/Base_Tests/data/native_libs.json +++ b/test/Base_Tests/data/native_libs.json @@ -125,18 +125,6 @@ "META-INF/resources/nfi-native/libnfi/darwin/aarch64/bin/libtrufflenfi.dylib", "META-INF/resources/nfi-native/libnfi/windows/amd64/bin/trufflenfi.dll" ], - "lib/Standard/Snowflake/0.0.0-dev/polyglot/java/conscrypt-openjdk-uber-2.5.2.jar": [ - "META-INF/native/conscrypt_openjdk_jni-windows-x86.dll", - "META-INF/native/conscrypt_openjdk_jni-windows-x86_64.dll", - "META-INF/native/libconscrypt_openjdk_jni-linux-x86_64.so", - "META-INF/native/libconscrypt_openjdk_jni-osx-x86_64.dylib" - ], - "lib/Standard/Google_Api/0.0.0-dev/polyglot/java/conscrypt-openjdk-uber-2.5.2.jar": [ - "META-INF/native/conscrypt_openjdk_jni-windows-x86.dll", - "META-INF/native/conscrypt_openjdk_jni-windows-x86_64.dll", - "META-INF/native/libconscrypt_openjdk_jni-linux-x86_64.so", - "META-INF/native/libconscrypt_openjdk_jni-osx-x86_64.dylib" - ], "lib/Standard/Microsoft/0.0.0-dev/polyglot/java/jna-wrapper-assembly-0.1.0-SNAPSHOT.jar": [ "com/sun/jna/freebsd-x86-64/libjnidispatch.so", "com/sun/jna/freebsd-x86/libjnidispatch.so",