Skip to content

Commit c5b8922

Browse files
wangxiaoyingwangxiaoying
authored
Merge pull request #808 from holicc/main
feat(postgres): add support for Inet type in Postgres integration fix #798
2 parents dd45a24 + c474f6a commit c5b8922

File tree

7 files changed

+61
-10
lines changed

7 files changed

+61
-10
lines changed

‎Cargo.lock

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎connectorx/Cargo.toml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ native-tls = {version = "0.2", optional = true}
3636
num-traits = {version = "0.2", optional = true}
3737
openssl = {version = "0.10", optional = true, features = ["vendored"]}
3838
oracle = {version = "0.6", optional = true}
39-
postgres = {version = "0.19", features = ["with-chrono-0_4", "with-uuid-1", "with-serde_json-1"], optional = true}
39+
postgres = {version = "0.19", features = ["with-chrono-0_4", "with-uuid-1", "with-serde_json-1","with-cidr-0_2"], optional = true}
4040
postgres-native-tls = {version = "0.5", optional = true}
4141
postgres-openssl = {version = "0.5", optional = true}
4242
mysql_common = {version = "0.32", features = ["chrono"], optional = true}
@@ -59,6 +59,7 @@ j4rs = {version = "0.22", optional = true}
5959
datafusion = {version = "46", optional = true}
6060
prusto = {version = "0.5", optional = true}
6161
serde = {version = "1", optional = true}
62+
cidr-02 = { version = "0.2", package = "cidr", optional = true }
6263

6364
[lib]
6465
crate-type = ["cdylib", "rlib"]
@@ -97,6 +98,7 @@ src_postgres = [
9798
"native-tls",
9899
"openssl",
99100
"postgres-openssl",
101+
"cidr-02",
100102
]
101103
src_sqlite = ["rusqlite", "r2d2_sqlite", "fallible-streaming-iterator", "r2d2", "urlencoding"]
102104
src_trino = ["prusto", "uuid", "urlencoding", "rust_decimal", "tokio", "num-traits", "serde"]

‎connectorx/src/sources/postgres/mod.rs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ mod errors;
55
mod typesystem;
66

77
pub use self::errors::PostgresSourceError;
8+
use cidr_02::IpInet;
89
pub use connection::rewrite_tls_args;
910
pub use typesystem::{PostgresTypePairs, PostgresTypeSystem};
1011

@@ -480,6 +481,7 @@ impl_produce!(
480481
NaiveTime,
481482
Uuid,
482483
Value,
484+
IpInet,
483485
Vec<Option<bool>>,
484486
Vec<Option<i16>>,
485487
Vec<Option<i32>>,
@@ -704,7 +706,7 @@ macro_rules! impl_csv_produce {
704706
};
705707
}
706708

707-
impl_csv_produce!(i8, i16, i32, i64, f32, f64, Uuid,);
709+
impl_csv_produce!(i8, i16, i32, i64, f32, f64, Uuid, IpInet,);
708710

709711
macro_rules! impl_csv_vec_produce {
710712
($($t: ty,)+) => {
@@ -1216,6 +1218,7 @@ impl_produce!(
12161218
NaiveTime,
12171219
Uuid,
12181220
Value,
1221+
IpInet,
12191222
HashMap<String, Option<String>>,
12201223
Vec<Option<bool>>,
12211224
Vec<Option<String>>,
@@ -1481,7 +1484,7 @@ macro_rules! impl_simple_produce {
14811484
};
14821485
}
14831486

1484-
impl_simple_produce!(i8, i16, i32, i64, f32, f64, Uuid,);
1487+
impl_simple_produce!(i8, i16, i32, i64, f32, f64, Uuid, IpInet,);
14851488

14861489
impl<'r> Produce<'r, bool> for PostgresSimpleSourceParser {
14871490
type Error = PostgresSourceError;

‎connectorx/src/sources/postgres/typesystem.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
2+
use cidr_02::IpInet;
23
use postgres::types::Type;
34
use rust_decimal::Decimal;
45
use serde_json::Value;
@@ -38,6 +39,7 @@ pub enum PostgresTypeSystem {
3839
Enum(bool),
3940
HSTORE(bool),
4041
Name(bool),
42+
Inet(bool),
4143
}
4244

4345
impl_typesystem! {
@@ -68,6 +70,7 @@ impl_typesystem! {
6870
{ UUID => Uuid }
6971
{ JSON | JSONB => Value }
7072
{ HSTORE => HashMap<String, Option<String>> }
73+
{ Inet => IpInet }
7174
}
7275
}
7376

@@ -104,6 +107,7 @@ impl<'a> From<&'a Type> for PostgresTypeSystem {
104107
"json" => JSON(true),
105108
"jsonb" => JSONB(true),
106109
"hstore" => HSTORE(true),
110+
"inet" => Inet(true),
107111
_ => match ty.kind() {
108112
postgres::types::Kind::Enum(_) => Enum(true),
109113
_ => unimplemented!("{}", ty.name()),

‎connectorx/src/transports/postgres_arrow.rs

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ use crate::sources::postgres::{
1212
};
1313
use crate::typesystem::TypeConversion;
1414
use chrono::{DateTime, NaiveDate, NaiveDateTime, NaiveTime, Utc};
15+
use cidr_02::IpInet;
1516
use num_traits::ToPrimitive;
1617
use postgres::NoTls;
1718
use postgres_openssl::MakeTlsConnector;
@@ -65,6 +66,7 @@ macro_rules! impl_postgres_transport {
6566
{ ByteA[Vec<u8>] => LargeBinary[Vec<u8>] | conversion auto }
6667
{ JSON[Value] => LargeUtf8[String] | conversion option }
6768
{ JSONB[Value] => LargeUtf8[String] | conversion none }
69+
{ Inet[IpInet] => LargeUtf8[String] | conversion none }
6870
{ BoolArray[Vec<Option<bool>>] => BoolArray[Vec<Option<bool>>] | conversion auto }
6971
{ VarcharArray[Vec<Option<String>>] => Utf8Array[Vec<Option<String>>] | conversion auto }
7072
{ TextArray[Vec<Option<String>>] => Utf8Array[Vec<Option<String>>] | conversion none }
@@ -88,6 +90,18 @@ impl_postgres_transport!(CursorProtocol, MakeTlsConnector);
8890
impl_postgres_transport!(SimpleProtocol, NoTls);
8991
impl_postgres_transport!(SimpleProtocol, MakeTlsConnector);
9092

93+
impl<P, C> TypeConversion<IpInet, String> for PostgresArrowTransport<P, C> {
94+
fn convert(val: IpInet) -> String {
95+
val.to_string()
96+
}
97+
}
98+
99+
impl<P, C> TypeConversion<Option<IpInet>, Option<String>> for PostgresArrowTransport<P, C> {
100+
fn convert(val: Option<IpInet>) -> Option<String> {
101+
val.map(|val| val.to_string())
102+
}
103+
}
104+
91105
impl<P, C> TypeConversion<NaiveTime, NaiveTimeWrapperMicro> for PostgresArrowTransport<P, C> {
92106
fn convert(val: NaiveTime) -> NaiveTimeWrapperMicro {
93107
NaiveTimeWrapperMicro(val)

‎connectorx/tests/test_postgres.rs

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -404,6 +404,7 @@ fn test_types_binary_postgres() {
404404
"test_varchararray",
405405
"test_textarray",
406406
"test_name",
407+
"test_inet",
407408
]
408409
.join(",");
409410

@@ -465,6 +466,7 @@ fn test_types_csv_postgres() {
465466
"test_varchararray",
466467
"test_textarray",
467468
"test_name",
469+
"test_inet",
468470
]
469471
.join(",");
470472

@@ -525,6 +527,7 @@ fn test_types_cursor_postgres() {
525527
"test_varchararray",
526528
"test_textarray",
527529
"test_name",
530+
"test_inet",
528531
]
529532
.join(",");
530533

@@ -583,6 +586,7 @@ fn test_types_simple_postgres() {
583586
"test_varchararray",
584587
"test_textarray",
585588
"test_name",
589+
"test_inet",
586590
]
587591
.join(",");
588592

@@ -1220,6 +1224,21 @@ pub fn verify_arrow_type_results(result: Vec<RecordBatch>, protocol: &str) {
12201224
Some("101203203-1212323-22131235"),
12211225
None,
12221226
])));
1227+
1228+
// test_inet
1229+
col += 1;
1230+
assert!(result[0]
1231+
.column(col)
1232+
.as_any()
1233+
.downcast_ref::<StringArray>()
1234+
.unwrap()
1235+
.eq(&StringArray::from(vec![
1236+
Some("192.168.1.1"),
1237+
Some("10.0.0.0/24"),
1238+
Some("2001:db8::1"),
1239+
Some("2001:db8::/32"),
1240+
None,
1241+
])));
12231242
}
12241243

12251244
#[test]

‎scripts/postgres.sql

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -87,15 +87,16 @@ CREATE TABLE IF NOT EXISTS test_types(
8787
test_ltxtquery ltxtquery,
8888
test_varchararray VARCHAR[],
8989
test_textarray TEXT[],
90-
test_name NAME
90+
test_name NAME,
91+
test_inet INET
9192
);
9293

93-
/* test_bool test_date test_timestamp test_timestamptz test_int2 test_int4 test_int8 test_float4 test_float8 test_numeric test_bpchar test_char test_varchar test_uuid test_time test_interval test_json test_jsonb test_bytea test_enum test_f4array test_f8array test_narray test_boolarray test_i2array test_i4array test_i8array test_citext test_ltree test_lquery test_ltxtquery test_varchararray test_textarray test_name */
94-
INSERT INTO test_types VALUES ( TRUE, '1970-01-01', '1970-01-01 00:00:01', '1970-01-01 00:00:01-00', -32768, 0, -9223372036854775808, -1.1, -1.1, .01, '👨‍🍳', 'a', 'abcdefghij', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', '08:12:40', '1 year 2 months 3 days', '{"customer": "John Doe", "items": {"product": "Beer", "qty": 6}}', '{"customer": "John Doe", "items": {"product": "Beer", "qty": 6}}', '\010', 'happy', '{-1.1, 0.00}', '{-1.1, 0.00}', '{0.01, 521.23}', '{true, false}', '{12}', '{-1}', '{-9223372036854775808, 9223372036854775807}', 'str_citext', 'A.B.C.D', '*.B.*', 'A & B*', ARRAY['str1','str2'], ARRAY['str1','str2'], '0' );
95-
INSERT INTO test_types VALUES ( true, '2000-02-28', '2000-02-28 12:00:10', '2000-02-28 12:00:10-04', 0, 1, 0, 0.00, 0.0000, 521.34, 'bb', 'ಠ', '', 'a0ee-bc99-9c0b-4ef8-bb6d-6bb9-bd38-0a11', '18:30:00', '2 weeks ago', '{"customer": "Lily Bush", "items": {"product": "Diaper", "qty": 24}}', '{"customer": "Lily Bush", "items": {"product": "Diaper", "qty": 24}}', 'Здра́вствуйте', 'very happy', '{}', '{}', '{0.12, 333.33, 22.22}', '{}', '{}', '{}', '{}', '', 'A.B.E', 'A.*', 'A | B', '{"0123456789","abcdefghijklmnopqrstuvwxyz","!@#$%^&*()_-+=~`:;<>?/"}', '{"0123456789","abcdefghijklmnopqrstuvwxyz","!@#$%^&*()_-+=~`:;<>?/"}', '21' );
96-
INSERT INTO test_types VALUES ( false, '2038-01-18', '2038-01-18 23:59:59', '2038-01-18 23:59:59+08', 1, -2147483648, 9223372036854775807, 2.123456, 2.12345678901, '1e-5', '', '😃', '👨‍🍳👨‍🍳👨‍🍳👨', 'A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11', '23:00:10', '3 months 2 days ago', '{"customer": "Josh William", "items": {"product": "Toy Car", "qty": 1}}', '{"customer": "Josh William", "items": {"product": "Toy Car", "qty": 1}}', '', 'ecstatic', '{1, -2, -12345.1}', '{2.12345678901, -12345678901.1}', '{}', '{true}', '{-32768, 32767}', '{-2147483648, 2147483647}', '{0}', 'abcdef', 'A', '*', 'A@', ARRAY['',' '], ARRAY['',' '], 'someName' );
97-
INSERT INTO test_types VALUES ( False, '1901-12-14', '1901-12-14 00:00:00.062547', '1901-12-14 00:00:00.062547-12', 32767, 2147483647, 1, -12345.1, -12345678901.1, -1.123e2, 'ddddd', '@', '@', '{a0eebc999c0b4ef8bb6d6bb9bd380a11}', '00:00:59.062547', '1 year 2 months 3 days', '{}', '{}', '😜', 'ecstatic', '{2.123456, NULL, 123.123}', '{2.123456, NULL, 123.123}', '{0.0, NULL, -112.1}', '{true, false, NULL}', '{-1, 0, 1, NULL}', '{-1, 0, 1123, NULL}', '{-1, 0, 1, NULL}', '1234', '', '*.A', 'A & B*', ARRAY['👨‍🍳👨‍🍳👨‍🍳👨','', NULL], ARRAY['👨‍🍳👨‍🍳👨‍🍳👨','', NULL], '101203203-1212323-22131235');
98-
INSERT INTO test_types VALUES ( NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL );
94+
/* test_bool test_date test_timestamp test_timestamptz test_int2 test_int4 test_int8 test_float4 test_float8 test_numeric test_bpchar test_char test_varchar test_uuid test_time test_interval test_json test_jsonb test_bytea test_enum test_f4array test_f8array test_narray test_boolarray test_i2array test_i4array test_i8array test_citext test_ltree test_lquery test_ltxtquery test_varchararray test_textarray test_name test_inet */
95+
INSERT INTO test_types VALUES ( TRUE, '1970-01-01', '1970-01-01 00:00:01', '1970-01-01 00:00:01-00', -32768, 0, -9223372036854775808, -1.1, -1.1, .01, '👨‍🍳', 'a', 'abcdefghij', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', '08:12:40', '1 year 2 months 3 days', '{"customer": "John Doe", "items": {"product": "Beer", "qty": 6}}', '{"customer": "John Doe", "items": {"product": "Beer", "qty": 6}}', '\010', 'happy', '{-1.1, 0.00}', '{-1.1, 0.00}', '{0.01, 521.23}', '{true, false}', '{12}', '{-1}', '{-9223372036854775808, 9223372036854775807}', 'str_citext', 'A.B.C.D', '*.B.*', 'A & B*', ARRAY['str1','str2'], ARRAY['str1','str2'], '0', '192.168.1.1' );
96+
INSERT INTO test_types VALUES ( true, '2000-02-28', '2000-02-28 12:00:10', '2000-02-28 12:00:10-04', 0, 1, 0, 0.00, 0.0000, 521.34, 'bb', 'ಠ', '', 'a0ee-bc99-9c0b-4ef8-bb6d-6bb9-bd38-0a11', '18:30:00', '2 weeks ago', '{"customer": "Lily Bush", "items": {"product": "Diaper", "qty": 24}}', '{"customer": "Lily Bush", "items": {"product": "Diaper", "qty": 24}}', 'Здра́вствуйте', 'very happy', '{}', '{}', '{0.12, 333.33, 22.22}', '{}', '{}', '{}', '{}', '', 'A.B.E', 'A.*', 'A | B', '{"0123456789","abcdefghijklmnopqrstuvwxyz","!@#$%^&*()_-+=~`:;<>?/"}', '{"0123456789","abcdefghijklmnopqrstuvwxyz","!@#$%^&*()_-+=~`:;<>?/"}', '21', '10.0.0.0/24' );
97+
INSERT INTO test_types VALUES ( false, '2038-01-18', '2038-01-18 23:59:59', '2038-01-18 23:59:59+08', 1, -2147483648, 9223372036854775807, 2.123456, 2.12345678901, '1e-5', '', '😃', '👨‍🍳👨‍🍳👨‍🍳👨', 'A0EEBC99-9C0B-4EF8-BB6D-6BB9BD380A11', '23:00:10', '3 months 2 days ago', '{"customer": "Josh William", "items": {"product": "Toy Car", "qty": 1}}', '{"customer": "Josh William", "items": {"product": "Toy Car", "qty": 1}}', '', 'ecstatic', '{1, -2, -12345.1}', '{2.12345678901, -12345678901.1}', '{}', '{true}', '{-32768, 32767}', '{-2147483648, 2147483647}', '{0}', 'abcdef', 'A', '*', 'A@', ARRAY['',' '], ARRAY['',' '], 'someName', '2001:db8::1' );
98+
INSERT INTO test_types VALUES ( False, '1901-12-14', '1901-12-14 00:00:00.062547', '1901-12-14 00:00:00.062547-12', 32767, 2147483647, 1, -12345.1, -12345678901.1, -1.123e2, 'ddddd', '@', '@', '{a0eebc999c0b4ef8bb6d6bb9bd380a11}', '00:00:59.062547', '1 year 2 months 3 days', '{}', '{}', '😜', 'ecstatic', '{2.123456, NULL, 123.123}', '{2.123456, NULL, 123.123}', '{0.0, NULL, -112.1}', '{true, false, NULL}', '{-1, 0, 1, NULL}', '{-1, 0, 1123, NULL}', '{-1, 0, 1, NULL}', '1234', '', '*.A', 'A & B*', ARRAY['👨‍🍳👨‍🍳👨‍🍳👨','', NULL], ARRAY['👨‍🍳👨‍🍳👨‍🍳👨','', NULL], '101203203-1212323-22131235', '2001:db8::/32' );
99+
INSERT INTO test_types VALUES ( NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL );
99100

100101

101102
CREATE OR REPLACE FUNCTION increment(i integer) RETURNS integer AS $$

0 commit comments

Comments
 (0)