Skip to content

Commit 29ae5ee

Browse files
authored
Extract conscrypt native libs from Snowflake and Google_Api (#13211)
* Remove expected constrypt native libs from both Google_Api and Snowflake * Extract conscrypt native libs within Google_Api * Extract conscrypt native libs within Snowflake * fmt
1 parent 531a90d commit 29ae5ee

File tree

4 files changed

+125
-14
lines changed

4 files changed

+125
-14
lines changed

build.sbt

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5271,7 +5271,7 @@ lazy val `std-google-api` = project
52715271
unmanagedClasspath = (Compile / unmanagedJars).value,
52725272
previousRun = prev
52735273
)
5274-
StdBits
5274+
val grpc = StdBits
52755275
.extractNativeLibsFromGrpc(
52765276
`google-api-polyglot-root`,
52775277
`google-api-native-libs`,
@@ -5283,6 +5283,18 @@ lazy val `std-google-api` = project
52835283
cacheStoreFactory = cacheStoreFactory,
52845284
previousRun = prev
52855285
)
5286+
val conscrypt = StdBits
5287+
.extractNativeLibsFromConscrypt(
5288+
`google-api-polyglot-root`,
5289+
`google-api-native-libs`,
5290+
updateReport = (Compile / update).value,
5291+
logger = streams.value.log,
5292+
moduleName = moduleName.value,
5293+
scalaBinaryVersion = scalaBinaryVersion.value,
5294+
cacheStoreFactory = cacheStoreFactory,
5295+
previousRun = prev
5296+
)
5297+
grpc.appended(conscrypt)
52865298
}.value,
52875299
cleanPolyglotRoot := Def.task {
52885300
import sbt.util.CacheImplicits._
@@ -5462,7 +5474,7 @@ lazy val `std-snowflake` = project
54625474
unmanagedClasspath = (Compile / unmanagedJars).value,
54635475
previousRun = prev
54645476
)
5465-
StdBits
5477+
val grpc = StdBits
54665478
.extractNativeLibsFromGrpc(
54675479
`std-snowflake-polyglot-root`,
54685480
`std-snowflake-native-libs`,
@@ -5474,6 +5486,18 @@ lazy val `std-snowflake` = project
54745486
cacheStoreFactory = cacheStoreFactory,
54755487
previousRun = prev
54765488
)
5489+
val conscrypt = StdBits
5490+
.extractNativeLibsFromConscrypt(
5491+
`std-snowflake-polyglot-root`,
5492+
`std-snowflake-native-libs`,
5493+
updateReport = (Compile / update).value,
5494+
logger = streams.value.log,
5495+
moduleName = moduleName.value,
5496+
scalaBinaryVersion = scalaBinaryVersion.value,
5497+
cacheStoreFactory = cacheStoreFactory,
5498+
previousRun = prev
5499+
)
5500+
grpc.appended(conscrypt)
54775501
}.value,
54785502
cleanPolyglotRoot := Def.task {
54795503
import sbt.util.CacheImplicits._

project/AnalysisOfExtractedNativeLibs.scala

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,12 @@ case class AnalysisOfExtractedNativeLibs(
1414
libs.find(_.from == srcJar)
1515

1616
def isOutdated: Boolean = libs.exists(_.isOutdated)
17+
18+
def appended(
19+
other: AnalysisOfExtractedNativeLibs
20+
): AnalysisOfExtractedNativeLibs = {
21+
AnalysisOfExtractedNativeLibs(libs ++ other.libs)
22+
}
1723
}
1824
object AnalysisOfExtractedNativeLibs {
1925
import sjsonnew.{:*:, LList, LNil}

project/StdBits.scala

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -506,6 +506,99 @@ object StdBits {
506506
)
507507
}
508508

509+
/** Extract native libraries from `org.conscrypt:conscrypt-openjdk-uber:2.5.2` jar, which is
510+
* a transitive dependency of
511+
* `com.google.analytics:google-analytics-admin:0.66.0` and of
512+
* `net.snowflake:snowflake-jdbc-thin:3.15.0`.
513+
*
514+
* Currently, it is included in both `Standard.Google_Api` and `Standard.Snowflake` libraries.
515+
*
516+
* Names of the native libraries in jar:
517+
* - `META-INF/native/conscrypt_openjdk_jni-windows-x86.dll`
518+
* - `META-INF/native/conscrypt_openjdk_jni-windows-x86_64.dll`
519+
* - `META-INF/native/libconscrypt_openjdk_jni-linux-x86_64.so`
520+
* - `META-INF/native/libconscrypt_openjdk_jni-osx-x86_64.dylib`
521+
*
522+
* The jar is signed, so we also have to remove `META-INF/SIGNING.SF`.
523+
*/
524+
def extractNativeLibsFromConscrypt(
525+
polyglotRootDir: File,
526+
nativeLibsDir: File,
527+
updateReport: UpdateReport,
528+
logger: ManagedLogger,
529+
moduleName: String,
530+
scalaBinaryVersion: String,
531+
cacheStoreFactory: CacheStoreFactory,
532+
previousRun: Option[AnalysisOfExtractedNativeLibs]
533+
): AnalysisOfExtractedNativeLibs = {
534+
if (previousRun.exists(!_.isOutdated)) {
535+
return previousRun.get
536+
}
537+
val osName = plainOsName().replace("macos", "osx")
538+
val validOsExt = osExt()
539+
val validArch = arch().replace("-", "_")
540+
val prefix = "META-INF/native"
541+
val entriesToRemove = Seq(
542+
"META-INF/SIGNINGC.SF",
543+
"META-INF/SIGNINGC.RSA"
544+
)
545+
val conscryptVersion = "2.5.2"
546+
547+
def renameFunc(entryName: String): Option[String] = {
548+
val strippedEntryName = entryName.substring(prefix.length + 1)
549+
val pattern = "^(.+)-(\\w+)-([\\w_]+)(\\.\\w+)$".r
550+
strippedEntryName match {
551+
case pattern(libname, entryOs, entryArch, entryExt) =>
552+
if (
553+
!entryOs.equals(osName) ||
554+
!entryArch.equals(validArch) ||
555+
!entryExt.equals(validOsExt)
556+
) {
557+
None
558+
} else {
559+
val outputArch = validArch.replace("x86_64", "amd64")
560+
Some(s"$outputArch/$osName/$libname$entryExt")
561+
}
562+
case _ =>
563+
throw new RuntimeException(
564+
s"Unexpected entry name format: $strippedEntryName"
565+
)
566+
}
567+
}
568+
569+
val conscryptJar = JPMSUtils
570+
.filterModulesFromUpdate(
571+
updateReport,
572+
Seq("org.conscrypt" % "conscrypt-openjdk-uber" % conscryptVersion),
573+
logger,
574+
moduleName,
575+
scalaBinaryVersion,
576+
shouldContainAll = true
577+
)
578+
.head
579+
val outputJar =
580+
(polyglotRootDir / s"conscrypt-openjdk-uber-$conscryptVersion.jar").toPath
581+
val extractedLibs = JARUtils.extractFilesFromJar(
582+
conscryptJar.toPath,
583+
Some(prefix),
584+
Some(outputJar),
585+
nativeLibsDir.toPath,
586+
renameFunc,
587+
logger,
588+
cacheStoreFactory,
589+
previousRun.flatMap(_.forJar(conscryptJar))
590+
)
591+
JARUtils.removeEntriesFromJar(
592+
outputJar,
593+
entryName => entriesToRemove.contains(entryName)
594+
)
595+
AnalysisOfExtractedNativeLibs(
596+
conscryptJar,
597+
extractedLibs.getOrElse(Nil),
598+
Some(outputJar.toFile)
599+
)
600+
}
601+
509602
def ensureDirExistsAndIsClean(
510603
path: Path,
511604
logger: sbt.util.Logger,

test/Base_Tests/data/native_libs.json

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -125,18 +125,6 @@
125125
"META-INF/resources/nfi-native/libnfi/darwin/aarch64/bin/libtrufflenfi.dylib",
126126
"META-INF/resources/nfi-native/libnfi/windows/amd64/bin/trufflenfi.dll"
127127
],
128-
"lib/Standard/Snowflake/0.0.0-dev/polyglot/java/conscrypt-openjdk-uber-2.5.2.jar": [
129-
"META-INF/native/conscrypt_openjdk_jni-windows-x86.dll",
130-
"META-INF/native/conscrypt_openjdk_jni-windows-x86_64.dll",
131-
"META-INF/native/libconscrypt_openjdk_jni-linux-x86_64.so",
132-
"META-INF/native/libconscrypt_openjdk_jni-osx-x86_64.dylib"
133-
],
134-
"lib/Standard/Google_Api/0.0.0-dev/polyglot/java/conscrypt-openjdk-uber-2.5.2.jar": [
135-
"META-INF/native/conscrypt_openjdk_jni-windows-x86.dll",
136-
"META-INF/native/conscrypt_openjdk_jni-windows-x86_64.dll",
137-
"META-INF/native/libconscrypt_openjdk_jni-linux-x86_64.so",
138-
"META-INF/native/libconscrypt_openjdk_jni-osx-x86_64.dylib"
139-
],
140128
"lib/Standard/Microsoft/0.0.0-dev/polyglot/java/jna-wrapper-assembly-0.1.0-SNAPSHOT.jar": [
141129
"com/sun/jna/freebsd-x86-64/libjnidispatch.so",
142130
"com/sun/jna/freebsd-x86/libjnidispatch.so",

0 commit comments

Comments
 (0)