Skip to content

Extract conscrypt native libs from Snowflake and Google_Api #13211

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jun 3, 2025
+125 −14
Merged

Extract conscrypt native libs from Snowflake and Google_Api #13211

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
97e9a6c
Remove expected constrypt native libs from both Google_Api and Snowflake
Akirathan Jun 2, 2025
e66818a
Extract conscrypt native libs within Google_Api
Akirathan Jun 2, 2025
8cbc74f
Extract conscrypt native libs within Snowflake
Akirathan Jun 2, 2025
43cb265
fmt
Akirathan Jun 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 26 additions & 2 deletions build.sbt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5271,7 +5271,7 @@ lazy val `std-google-api` = project
unmanagedClasspath = (Compile / unmanagedJars).value,
previousRun = prev
)
StdBits
val grpc = StdBits
.extractNativeLibsFromGrpc(
`google-api-polyglot-root`,
`google-api-native-libs`,
Expand All @@ -5283,6 +5283,18 @@ lazy val `std-google-api` = project
cacheStoreFactory = cacheStoreFactory,
previousRun = prev
)
val conscrypt = StdBits
.extractNativeLibsFromConscrypt(
`google-api-polyglot-root`,
`google-api-native-libs`,
updateReport = (Compile / update).value,
logger = streams.value.log,
moduleName = moduleName.value,
scalaBinaryVersion = scalaBinaryVersion.value,
cacheStoreFactory = cacheStoreFactory,
previousRun = prev
)
grpc.appended(conscrypt)
}.value,
cleanPolyglotRoot := Def.task {
import sbt.util.CacheImplicits._
Expand Down Expand Up @@ -5462,7 +5474,7 @@ lazy val `std-snowflake` = project
unmanagedClasspath = (Compile / unmanagedJars).value,
previousRun = prev
)
StdBits
val grpc = StdBits
.extractNativeLibsFromGrpc(
`std-snowflake-polyglot-root`,
`std-snowflake-native-libs`,
Expand All @@ -5474,6 +5486,18 @@ lazy val `std-snowflake` = project
cacheStoreFactory = cacheStoreFactory,
previousRun = prev
)
val conscrypt = StdBits
.extractNativeLibsFromConscrypt(
`std-snowflake-polyglot-root`,
`std-snowflake-native-libs`,
updateReport = (Compile / update).value,
logger = streams.value.log,
moduleName = moduleName.value,
scalaBinaryVersion = scalaBinaryVersion.value,
cacheStoreFactory = cacheStoreFactory,
previousRun = prev
)
grpc.appended(conscrypt)
}.value,
cleanPolyglotRoot := Def.task {
import sbt.util.CacheImplicits._
Expand Down
6 changes: 6 additions & 0 deletions project/AnalysisOfExtractedNativeLibs.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@ case class AnalysisOfExtractedNativeLibs(
libs.find(_.from == srcJar)

def isOutdated: Boolean = libs.exists(_.isOutdated)

def appended(
other: AnalysisOfExtractedNativeLibs
): AnalysisOfExtractedNativeLibs = {
AnalysisOfExtractedNativeLibs(libs ++ other.libs)
}
}
object AnalysisOfExtractedNativeLibs {
import sjsonnew.{:*:, LList, LNil}
Expand Down
93 changes: 93 additions & 0 deletions project/StdBits.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -506,6 +506,99 @@ object StdBits {
)
}

/** Extract native libraries from `org.conscrypt:conscrypt-openjdk-uber:2.5.2` jar, which is
* a transitive dependency of
* `com.google.analytics:google-analytics-admin:0.66.0` and of
* `net.snowflake:snowflake-jdbc-thin:3.15.0`.
*
* Currently, it is included in both `Standard.Google_Api` and `Standard.Snowflake` libraries.
*
* Names of the native libraries in jar:
* - `META-INF/native/conscrypt_openjdk_jni-windows-x86.dll`
* - `META-INF/native/conscrypt_openjdk_jni-windows-x86_64.dll`
* - `META-INF/native/libconscrypt_openjdk_jni-linux-x86_64.so`
* - `META-INF/native/libconscrypt_openjdk_jni-osx-x86_64.dylib`
*
* The jar is signed, so we also have to remove `META-INF/SIGNING.SF`.
*/
def extractNativeLibsFromConscrypt(
polyglotRootDir: File,
nativeLibsDir: File,
updateReport: UpdateReport,
logger: ManagedLogger,
moduleName: String,
scalaBinaryVersion: String,
cacheStoreFactory: CacheStoreFactory,
previousRun: Option[AnalysisOfExtractedNativeLibs]
): AnalysisOfExtractedNativeLibs = {
if (previousRun.exists(!_.isOutdated)) {
return previousRun.get
}
val osName = plainOsName().replace("macos", "osx")
val validOsExt = osExt()
val validArch = arch().replace("-", "_")
val prefix = "META-INF/native"
val entriesToRemove = Seq(
"META-INF/SIGNINGC.SF",
"META-INF/SIGNINGC.RSA"
)
val conscryptVersion = "2.5.2"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should be provided as a parameter of a function. We don't want to have multiple versions of conscrypt floating around.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure I agree on this one. As mentioned in the docs for this method, org.constrypt:constrypt-openjdk-uber:2.5.2 is a transitive dependency. It is not directly listed by us anywhere. Besides, if some of our direct dependencies changes version of this transitive dependency, JPMSUtils.filterModulesFromUpdate method call will fail.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK. In that case it is still broken. If we upgrade the dependency that brings conscrypt and the former bumps its version, it will only show up when running your test suite (I guess?). Ideally the inference of the version would be automatic.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it will only show up when running your test suite (I guess?)

This is no "test suite". This is done during buildEngineDistribution. So if anything upgrades the transitive dependency, there will be an early failure - buildEngineDistribution will be failing.

Ideally the inference of the version would be automatic.

Yes, ideally it should. Ideally, sbt should be properly documented and it should expose API for fetching dependencies. But it does not.


def renameFunc(entryName: String): Option[String] = {
val strippedEntryName = entryName.substring(prefix.length + 1)
val pattern = "^(.+)-(\\w+)-([\\w_]+)(\\.\\w+)$".r
strippedEntryName match {
case pattern(libname, entryOs, entryArch, entryExt) =>
if (
!entryOs.equals(osName) ||
!entryArch.equals(validArch) ||
!entryExt.equals(validOsExt)
) {
None
} else {
val outputArch = validArch.replace("x86_64", "amd64")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why this?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the name of the directory that will be put inside the polyglot/lib/<osname> directory. The expected directory structure inside polyglot/lib is documented in NativeLibraryFinder.

Some(s"$outputArch/$osName/$libname$entryExt")
}
case _ =>
throw new RuntimeException(
s"Unexpected entry name format: $strippedEntryName"
)
}
}

val conscryptJar = JPMSUtils
.filterModulesFromUpdate(
updateReport,
Seq("org.conscrypt" % "conscrypt-openjdk-uber" % conscryptVersion),
logger,
moduleName,
scalaBinaryVersion,
shouldContainAll = true
)
.head
val outputJar =
(polyglotRootDir / s"conscrypt-openjdk-uber-$conscryptVersion.jar").toPath
val extractedLibs = JARUtils.extractFilesFromJar(
conscryptJar.toPath,
Some(prefix),
Some(outputJar),
nativeLibsDir.toPath,
renameFunc,
logger,
cacheStoreFactory,
previousRun.flatMap(_.forJar(conscryptJar))
)
JARUtils.removeEntriesFromJar(
outputJar,
entryName => entriesToRemove.contains(entryName)
)
AnalysisOfExtractedNativeLibs(
conscryptJar,
extractedLibs.getOrElse(Nil),
Some(outputJar.toFile)
)
}

def ensureDirExistsAndIsClean(
path: Path,
logger: sbt.util.Logger,
Expand Down
12 changes: 0 additions & 12 deletions test/Base_Tests/data/native_libs.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,18 +125,6 @@
"META-INF/resources/nfi-native/libnfi/darwin/aarch64/bin/libtrufflenfi.dylib",
"META-INF/resources/nfi-native/libnfi/windows/amd64/bin/trufflenfi.dll"
],
"lib/Standard/Snowflake/0.0.0-dev/polyglot/java/conscrypt-openjdk-uber-2.5.2.jar": [
"META-INF/native/conscrypt_openjdk_jni-windows-x86.dll",
"META-INF/native/conscrypt_openjdk_jni-windows-x86_64.dll",
"META-INF/native/libconscrypt_openjdk_jni-linux-x86_64.so",
"META-INF/native/libconscrypt_openjdk_jni-osx-x86_64.dylib"
],
"lib/Standard/Google_Api/0.0.0-dev/polyglot/java/conscrypt-openjdk-uber-2.5.2.jar": [
"META-INF/native/conscrypt_openjdk_jni-windows-x86.dll",
"META-INF/native/conscrypt_openjdk_jni-windows-x86_64.dll",
"META-INF/native/libconscrypt_openjdk_jni-linux-x86_64.so",
"META-INF/native/libconscrypt_openjdk_jni-osx-x86_64.dylib"
],
"lib/Standard/Microsoft/0.0.0-dev/polyglot/java/jna-wrapper-assembly-0.1.0-SNAPSHOT.jar": [
"com/sun/jna/freebsd-x86-64/libjnidispatch.so",
"com/sun/jna/freebsd-x86/libjnidispatch.so",
Expand Down
Loading