Skip to content

Commit 43c3584

Browse files
authored
fix: handle big integers in incoming events (cloudevents#495)
* fix: handle big integers in incoming events An event may have data that contains a BigInt. The builtin `JSON` parser for JavaScript does not handle the `BigInt` types. The introduced `json-bigint` dependency (34k) does. Fixes: cloudevents#489 Signed-off-by: Lance Ball <lball@redhat.com>
1 parent 1995b57 commit 43c3584

File tree

7 files changed

+562
-434
lines changed

7 files changed

+562
-434
lines changed

README.md

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,20 @@ const event3 = new CloudEvent({
162162
});
163163
```
164164

165+
### A Note About Big Integers
166+
167+
When parsing JSON data, if a JSON field value is a number, and that number
168+
is really big, JavaScript loses precision. For example, the Twitter API exposes
169+
the Tweet ID. This is a large number that exceeds the integer space of `Number`.
170+
171+
In order to address this situation, you can set the environment variable
172+
`CE_USE_BIG_INT` to the string value `"true"` to enable the use of the
173+
[`json-bigint`](https://www.npmjs.com/package/json-bigint) package. This
174+
package is not used by default due to the resulting slowdown in parse speed
175+
by a factor of 7x.
176+
177+
See for more information: https://github.com/cloudevents/sdk-javascript/issues/489
178+
165179
### Example Applications
166180

167181
There are a few trivial example applications in
@@ -205,7 +219,7 @@ There you will find Express.js, TypeScript and Websocket examples.
205219
| HTTP Batch | :heavy_check_mark: | :heavy_check_mark: |
206220
| Kafka Binary | :heavy_check_mark: | :heavy_check_mark: |
207221
| Kafka Structured | :heavy_check_mark: | :heavy_check_mark: |
208-
| Kafka Batch | :heavy_check_mark: | :heavy_check_mark:
222+
| Kafka Batch | :heavy_check_mark: | :heavy_check_mark:
209223
| MQTT Binary | :heavy_check_mark: | :heavy_check_mark: |
210224
| MQTT Structured | :heavy_check_mark: | :heavy_check_mark: |
211225

package-lock.json

Lines changed: 514 additions & 432 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@
113113
"ajv": "^8.11.0",
114114
"ajv-formats": "^2.1.1",
115115
"process": "^0.11.10",
116+
"json-bigint": "^1.0.0",
116117
"util": "^0.12.4",
117118
"uuid": "^8.3.2"
118119
},
@@ -121,6 +122,7 @@
121122
"@types/chai": "^4.2.11",
122123
"@types/cucumber": "^6.0.1",
123124
"@types/got": "^9.6.11",
125+
"@types/json-bigint": "^1.0.1",
124126
"@types/mocha": "^7.0.2",
125127
"@types/node": "^14.14.10",
126128
"@types/superagent": "^4.1.10",

src/constants.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ const CONSTANTS = Object.freeze({
5454
DATA_SCHEMA: "dataschema",
5555
DATA_BASE64: "data_base64",
5656
},
57+
USE_BIG_INT_ENV: "CE_USE_BIG_INT"
5758
} as const);
5859

5960
export default CONSTANTS;

src/parsers.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,11 @@
33
SPDX-License-Identifier: Apache-2.0
44
*/
55

6+
import JSONbig from "json-bigint";
67
import CONSTANTS from "./constants";
78
import { isString, isDefinedOrThrow, isStringOrObjectOrThrow, ValidationError } from "./event/validation";
89

10+
const __JSON = JSON;
911
export abstract class Parser {
1012
abstract parse(payload: Record<string, unknown> | string | string[] | undefined): unknown;
1113
}
@@ -36,6 +38,13 @@ export class JSONParser implements Parser {
3638

3739
isDefinedOrThrow(payload, new ValidationError("null or undefined payload"));
3840
isStringOrObjectOrThrow(payload, new ValidationError("invalid payload type, allowed are: string or object"));
41+
42+
if (process.env[CONSTANTS.USE_BIG_INT_ENV] === "true") {
43+
JSON = JSONbig(({ useNativeBigInt: true })) as JSON;
44+
} else {
45+
JSON = __JSON;
46+
}
47+
3948
const parseJSON = (v: Record<string, unknown> | string): string => (isString(v) ? JSON.parse(v as string) : v);
4049
return parseJSON(payload);
4150
}

test/integration/message_test.ts

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,26 @@ describe("HTTP transport", () => {
7575
expect(body.extboolean).to.equal(false);
7676
});
7777

78+
it("Handles big integers in structured mode", () => {
79+
process.env[CONSTANTS.USE_BIG_INT_ENV] = "true";
80+
const ce = HTTP.toEvent({
81+
headers: { "content-type": "application/cloudevents+json" },
82+
body: `{"data": 1524831183200260097}`
83+
}) as CloudEvent;
84+
expect(ce.data).to.equal(1524831183200260097n);
85+
process.env[CONSTANTS.USE_BIG_INT_ENV] = undefined;
86+
});
87+
88+
it("Handles big integers in binary mode", () => {
89+
process.env[CONSTANTS.USE_BIG_INT_ENV] = "true";
90+
const ce = HTTP.toEvent({
91+
headers: { "content-type": "application/json", "ce-id": "1234" },
92+
body: `{"data": 1524831183200260097}`
93+
}) as CloudEvent<Record<string, never>>;
94+
expect((ce.data as Record<string, never>).data).to.equal(1524831183200260097n);
95+
process.env[CONSTANTS.USE_BIG_INT_ENV] = undefined;
96+
});
97+
7898
it("Handles events with no content-type and no datacontenttype", () => {
7999
const body = "{Something[Not:valid}JSON";
80100
const message: Message<undefined> = {

tsconfig.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"compilerOptions": {
3-
"target": "ES2016", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */
3+
"target": "ES2020", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */
44
"module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */
55
"allowJs": true, /* Allow javascript files to be compiled. */
66
"checkJs": false, /* Report errors in .js files. */

0 commit comments

Comments
 (0)