Merge develop into master for release

tkaldahl_xirgo · tkaldahl_xirgo · commit cc2809394c37 · 2025-11-12T13:24:48.000-06:00
diff --git a/actions/pom.xml b/actions/pom.xml
@@ -25,7 +25,7 @@
     <parent>
         <groupId>io.em2m.sdk</groupId>
         <artifactId>em2m-java-sdk-pom</artifactId>
-        <version>2.92.0</version>
+        <version>2.93.0</version>
     </parent>
 
     <packaging>jar</packaging>
diff --git a/actions/src/main/java/io/em2m/actions/lambda/LambdaRuntime.kt b/actions/src/main/java/io/em2m/actions/lambda/LambdaRuntime.kt
@@ -7,6 +7,7 @@ import io.em2m.actions.model.ActionProcessor
 import io.em2m.actions.model.MultipartData
 import io.em2m.problem.Problem
 import io.em2m.policy.model.Claims
+import io.em2m.utils.parseCharset
 import java.util.*
 
 open class LambdaRuntime(
@@ -22,10 +23,13 @@ open class LambdaRuntime(
         // TODO: Detect and pars MultiPart
         val multipart: MultipartData? = null
 
+        val contentType = request.contentType
+        val charset = parseCharset(contentType)
+
         val context = ActionContext(
                 actionName = "$actionPrefix:$actionName",
                 claims = Claims(),
-                inputStream = request.body?.toByteArray()?.inputStream() ?: byteArrayOf().inputStream(),
+                inputStream = request.body?.toByteArray(charset)?.inputStream() ?: byteArrayOf().inputStream(),
                 environment = env.toMutableMap(),
                 multipart = multipart,
                 response = response)
diff --git a/actions/src/main/java/io/em2m/actions/xforms/JacksonRequestTransformer.kt b/actions/src/main/java/io/em2m/actions/xforms/JacksonRequestTransformer.kt
@@ -11,10 +11,12 @@ import io.em2m.actions.model.TypedActionFlow
 import io.em2m.problem.Problem
 import io.em2m.simplex.evalPath
 import io.em2m.utils.coerce
+import io.em2m.utils.parseCharset
 import org.xerial.snappy.SnappyInputStream
 import java.io.ByteArrayOutputStream
 import java.io.IOException
 import java.io.InputStream
+import java.nio.charset.Charset
 import java.util.*
 import java.util.zip.DeflaterInputStream
 import java.util.zip.GZIPInputStream
@@ -33,6 +35,8 @@ class JacksonRequestTransformer(
 
         val contentType = (ctx.environment["ContentType"] as? String ?: "").lowercase(Locale.getDefault())
 
+        val charset = parseCharset(contentType)
+
         if (contentType.contains("xml")) return
 
         try {
@@ -43,7 +47,7 @@ class JacksonRequestTransformer(
                     "snappy" -> SnappyInputStream(ctx.inputStream)
                     else -> ctx.inputStream
                 }
-                val sanitizedInputStream = sanitizeInputStream(inputStream)
+                val sanitizedInputStream = sanitizeInputStream(inputStream, charset)
                 val obj = objectMapper.readValue(sanitizedInputStream, type)
 //                val obj = objectMapper.readValue(inputStream, type)
                 ctx.request = obj
@@ -92,25 +96,14 @@ class JacksonRequestTransformer(
         }
     }
 
-    private fun sanitizeInputStream(inputStream: InputStream?): InputStream? {
+    private fun sanitizeInputStream(inputStream: InputStream?, charset: Charset = Charsets.UTF_8): InputStream? {
         if (inputStream == null) {
             return null
         }
 
-        val buffer = ByteArray(8192)
-        val outputStream = ByteArrayOutputStream()
-
-        inputStream.buffered().use { input ->
-            outputStream.use { output ->
-                var bytesRead: Int
-                while (input.read(buffer).also { bytesRead = it } != -1) {
-                    val sanitizedChunk = removeScriptTags(String(buffer, 0, bytesRead))
-                    output.write(sanitizedChunk.toByteArray())
-                }
-            }
-        }
-
-        return outputStream.toByteArray().inputStream()
+        val request = inputStream.reader(charset).use { it.readText() }
+        val sanitized = removeScriptTags(request)
+        return sanitized.byteInputStream(charset)
     }
 
     private fun removeScriptTags(input: String): String {
diff --git a/actions/src/test/java/io/em2m/actions/JacksonRequestTransformerTest.kt b/actions/src/test/java/io/em2m/actions/JacksonRequestTransformerTest.kt
@@ -35,10 +35,12 @@ class JacksonRequestTransformerTest {
             )
         )
 
+        val charset = Charsets.UTF_8
+
         val testContext = ActionContext(
             actionName = "TestService:TestAction",
             claims = Claims(),
-            inputStream = testRequest.body?.toByteArray()?.inputStream() ?: byteArrayOf().inputStream(),
+            inputStream = testRequest.body?.toByteArray(charset)?.inputStream() ?: byteArrayOf().inputStream(),
             response = LambdaResponse(),
             environment = mutableMapOf(
                 "ContentType" to "application/json"
diff --git a/ext/pom.xml b/ext/pom.xml
@@ -25,7 +25,7 @@
     <parent>
         <groupId>io.em2m.sdk</groupId>
         <artifactId>em2m-java-sdk-pom</artifactId>
-        <version>2.92.0</version>
+        <version>2.93.0</version>
     </parent>
 
     <packaging>jar</packaging>
diff --git a/geo/pom.xml b/geo/pom.xml
@@ -25,7 +25,7 @@
     <parent>
         <groupId>io.em2m.sdk</groupId>
         <artifactId>em2m-java-sdk-pom</artifactId>
-        <version>2.92.0</version>
+        <version>2.93.0</version>
     </parent>
 
     <packaging>jar</packaging>
diff --git a/policy/pom.xml b/policy/pom.xml
@@ -25,7 +25,7 @@
     <parent>
         <groupId>io.em2m.sdk</groupId>
         <artifactId>em2m-java-sdk-pom</artifactId>
-        <version>2.92.0</version>
+        <version>2.93.0</version>
     </parent>
 
     <packaging>jar</packaging>
diff --git a/pom.xml b/pom.xml
@@ -26,7 +26,7 @@
     <artifactId>em2m-java-sdk-pom</artifactId>
     <packaging>pom</packaging>
     <name>em2m-java-sdk-pom</name>
-    <version>2.92.0</version>
+    <version>2.93.0</version>
 
     <modules>
         <module>utils</module>
diff --git a/problem/pom.xml b/problem/pom.xml
@@ -25,12 +25,12 @@
     <parent>
         <groupId>io.em2m.sdk</groupId>
         <artifactId>em2m-java-sdk-pom</artifactId>
-        <version>2.92.0</version>
+        <version>2.93.0</version>
     </parent>
 
     <packaging>jar</packaging>
     <artifactId>em2m-java-sdk-problem</artifactId>
-    <version>2.92.0</version>
+    <version>2.93.0</version>
     <name>em2m-java-sdk-problem</name>
 
     <dependencies>
diff --git a/search/bean/pom.xml b/search/bean/pom.xml
@@ -25,7 +25,7 @@
     <parent>
         <groupId>io.em2m.sdk</groupId>
         <artifactId>em2m-java-sdk-pom</artifactId>
-        <version>2.92.0</version>
+        <version>2.93.0</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
diff --git a/search/core/pom.xml b/search/core/pom.xml
@@ -25,7 +25,7 @@
     <parent>
         <groupId>io.em2m.sdk</groupId>
         <artifactId>em2m-java-sdk-pom</artifactId>
-        <version>2.92.0</version>
+        <version>2.93.0</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
diff --git a/search/es/pom.xml b/search/es/pom.xml
@@ -25,7 +25,7 @@
     <parent>
         <groupId>io.em2m.sdk</groupId>
         <artifactId>em2m-java-sdk-pom</artifactId>
-        <version>2.92.0</version>
+        <version>2.93.0</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
diff --git a/search/mongo/pom.xml b/search/mongo/pom.xml
@@ -25,7 +25,7 @@
     <parent>
         <groupId>io.em2m.sdk</groupId>
         <artifactId>em2m-java-sdk-pom</artifactId>
-        <version>2.92.0</version>
+        <version>2.93.0</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
 
diff --git a/search/pom.xml b/search/pom.xml
@@ -25,7 +25,7 @@
     <parent>
         <groupId>io.em2m.sdk</groupId>
         <artifactId>em2m-java-sdk-pom</artifactId>
-        <version>2.92.0</version>
+        <version>2.93.0</version>
     </parent>
 
     <packaging>pom</packaging>
diff --git a/simplex/pom.xml b/simplex/pom.xml
@@ -25,7 +25,7 @@
     <parent>
         <groupId>io.em2m.sdk</groupId>
         <artifactId>em2m-java-sdk-pom</artifactId>
-        <version>2.92.0</version>
+        <version>2.93.0</version>
     </parent>
 
     <packaging>jar</packaging>
diff --git a/utils/pom.xml b/utils/pom.xml
@@ -25,12 +25,12 @@
     <parent>
         <groupId>io.em2m.sdk</groupId>
         <artifactId>em2m-java-sdk-pom</artifactId>
-        <version>2.92.0</version>
+        <version>2.93.0</version>
     </parent>
 
     <packaging>jar</packaging>
     <artifactId>em2m-java-sdk-utils</artifactId>
-    <version>2.92.0</version>
+    <version>2.93.0</version>
     <name>em2m-java-sdk-utils</name>
 
     <dependencies>
diff --git a/utils/src/main/java/io/em2m/utils/regex.kt b/utils/src/main/java/io/em2m/utils/regex.kt
@@ -0,0 +1,9 @@
+package io.em2m.utils
+
+fun Regex.firstIncompleteMatch(baseString: String): String? {
+    return this.find(baseString)?.groupValues?.getOrNull(1)
+}
+
+fun Regex.firstIncompleteMatchOr(baseString: String, default: String): String {
+    return this.firstIncompleteMatch(baseString) ?: default
+}
diff --git a/utils/src/main/java/io/em2m/utils/request.kt b/utils/src/main/java/io/em2m/utils/request.kt
@@ -0,0 +1,16 @@
+package io.em2m.utils
+
+import java.nio.charset.Charset
+
+private val DEFAULT_CHARSET = Charsets.UTF_8
+private val CHARSET_REGEX = "(?i)charset\\s*=\\s*['\"]?([^;'\"]+)".toRegex()
+
+fun parseCharset(contentType: String?): Charset {
+    if (contentType == null) return DEFAULT_CHARSET
+    val charsetString = CHARSET_REGEX.firstIncompleteMatchOr(contentType, "UTF-8")
+    return try {
+        Charset.forName(charsetString)
+    } catch (_: Exception) {
+        DEFAULT_CHARSET
+    }
+}
diff --git a/utils/src/test/java/io/em2m/utils/CharsetTest.kt b/utils/src/test/java/io/em2m/utils/CharsetTest.kt
@@ -0,0 +1,63 @@
+package io.em2m.utils
+
+import org.junit.Test
+import java.io.ByteArrayOutputStream
+
+class CharsetTest {
+
+    private fun brokenCode(characters: List<Char>): String {
+        val inputStream = characters.toString().byteInputStream()
+
+        val buffer = ByteArray(8192)
+        val outputStream = ByteArrayOutputStream()
+
+        inputStream.buffered().use { input ->
+            outputStream.use { output ->
+                var bytesRead: Int
+                while (input.read(buffer).also { bytesRead = it } != -1) {
+                    val sanitizedChunk = String(buffer, 0, bytesRead)
+                    output.write(sanitizedChunk.toByteArray())
+                }
+            }
+        }
+
+        return outputStream.toString()
+    }
+
+    private fun fixedCode(characters: List<Char>): String {
+        val charset = Charsets.UTF_8
+        val inputStream = characters.toString().byteInputStream(charset)
+        return inputStream.reader(charset).use { it.readText() }
+    }
+
+    @Test
+    fun `U+2019 Right Single Quotation Mark`() {
+        val characters = mutableListOf<Char>()
+        repeat(8191) { characters.add('a') }
+        // https://www.compart.com/en/unicode/U+2019
+        // Character represented as 3 bytes: 0xE2 0x80 0x99
+        characters.add('’')
+
+        val brokenRepresentation = brokenCode(characters)
+        val fixedRepresentation = fixedCode(characters)
+
+        assert('�' in brokenRepresentation)
+        assert('�' !in fixedRepresentation)
+    }
+
+    @Test
+    fun `U+2665 Heart Emoji`() {
+        val characters = mutableListOf<Char>()
+        repeat(8191) { characters.add('a') }
+        // https://www.compart.com/en/unicode/U+2665
+        // Character represented as 3 bytes: 0xE2 0x99 0xA5
+        characters.add('♥')
+
+        val brokenRepresentation = brokenCode(characters)
+        val fixedRepresentation = fixedCode(characters)
+
+        assert('�' in brokenRepresentation)
+        assert('�' !in fixedRepresentation)
+    }
+
+}

Original file line number	Diff line number	Diff line change
`@@ -35,10 +35,12 @@ class JacksonRequestTransformerTest {`
`35`	`35`	`)`
`36`	`36`	`)`
`37`	`37`
	`38`	`+ val charset = Charsets.UTF_8`
	`39`	`+`
`38`	`40`	`val testContext = ActionContext(`
`39`	`41`	`actionName = "TestService:TestAction",`
`40`	`42`	`claims = Claims(),`
`41`		`- inputStream = testRequest.body?.toByteArray()?.inputStream() ?: byteArrayOf().inputStream(),`
	`43`	`+ inputStream = testRequest.body?.toByteArray(charset)?.inputStream() ?: byteArrayOf().inputStream(),`
`42`	`44`	`response = LambdaResponse(),`
`43`	`45`	`environment = mutableMapOf(`
`44`	`46`	`"ContentType" to "application/json"`