From 27f5d765e3f9361fe932fe963eb49539d80aa112 Mon Sep 17 00:00:00 2001 From: Nassim Date: Wed, 19 Mar 2025 00:00:05 +0100 Subject: [PATCH 1/3] Make map encoding configurable --- lib/ecto/adapters/sqlite3.ex | 3 ++ lib/ecto/adapters/sqlite3/data_type.ex | 16 ++++++++-- test/ecto/adapters/sqlite3/data_type_test.exs | 14 +++++++-- test/ecto/integration/json_test.exs | 31 +++++++++++++++++-- 4 files changed, 58 insertions(+), 6 deletions(-) diff --git a/lib/ecto/adapters/sqlite3.ex b/lib/ecto/adapters/sqlite3.ex index 1aedb7a..9ee9d5b 100644 --- a/lib/ecto/adapters/sqlite3.ex +++ b/lib/ecto/adapters/sqlite3.ex @@ -51,6 +51,9 @@ defmodule Ecto.Adapters.SQLite3 do * `:uuid_type` - Defaults to `:string`. Determines the type of `:uuid` columns. Possible values and column types are the same as for [binary IDs](#module-binary-id-types). + * `:map_type` - Defaults to `:string`. Determines the type of `:map` columns. + Set to `:binary` to use the [JSONB](https://sqlite.org/jsonb.html) + storage format. * `:datetime_type` - Defaults to `:iso8601`. Determines how datetime fields are stored in the database. The allowed values are `:iso8601` and `:text_datetime`. `:iso8601` corresponds to a string of the form `YYYY-MM-DDThh:mm:ss` and diff --git a/lib/ecto/adapters/sqlite3/data_type.ex b/lib/ecto/adapters/sqlite3/data_type.ex index 0781d87..d021918 100644 --- a/lib/ecto/adapters/sqlite3/data_type.ex +++ b/lib/ecto/adapters/sqlite3/data_type.ex @@ -17,9 +17,7 @@ defmodule Ecto.Adapters.SQLite3.DataType do def column_type(:string, _opts), do: "TEXT" def column_type(:float, _opts), do: "NUMERIC" def column_type(:binary, _opts), do: "BLOB" - def column_type(:map, _opts), do: "TEXT" def column_type(:array, _opts), do: "TEXT" - def column_type({:map, _}, _opts), do: "TEXT" def column_type({:array, _}, _opts), do: "TEXT" def column_type(:date, _opts), do: "TEXT" def column_type(:utc_datetime, _opts), do: "TEXT" @@ -50,6 +48,20 @@ defmodule Ecto.Adapters.SQLite3.DataType do end end + def column_type(:map, _opts) do + case Application.get_env(:ecto_sqlite3, :map_type, :string) do + :string -> "TEXT" + :binary -> "BLOB" + end + end + + def column_type({:map, _}, _opts) do + case Application.get_env(:ecto_sqlite3, :map_type, :string) do + :string -> "TEXT" + :binary -> "BLOB" + end + end + def column_type(:uuid, _opts) do case Application.get_env(:ecto_sqlite3, :uuid_type, :string) do :string -> "TEXT" diff --git a/test/ecto/adapters/sqlite3/data_type_test.exs b/test/ecto/adapters/sqlite3/data_type_test.exs index 30d7044..6ce8979 100644 --- a/test/ecto/adapters/sqlite3/data_type_test.exs +++ b/test/ecto/adapters/sqlite3/data_type_test.exs @@ -5,10 +5,12 @@ defmodule Ecto.Adapters.SQLite3.DataTypeTest do setup do Application.put_env(:ecto_sqlite3, :binary_id_type, :string) + Application.put_env(:ecto_sqlite3, :map_type, :string) Application.put_env(:ecto_sqlite3, :uuid_type, :string) on_exit(fn -> Application.put_env(:ecto_sqlite3, :binary_id_type, :string) + Application.put_env(:ecto_sqlite3, :map_type, :string) Application.put_env(:ecto_sqlite3, :uuid_type, :string) end) end @@ -46,12 +48,20 @@ defmodule Ecto.Adapters.SQLite3.DataTypeTest do assert DataType.column_type(:uuid, nil) == "BLOB" end - test ":map is TEXT" do + test ":map is TEXT or BLOB" do assert DataType.column_type(:map, nil) == "TEXT" + + Application.put_env(:ecto_sqlite3, :map_type, :binary) + + assert DataType.column_type(:map, nil) == "BLOB" end - test "{:map, _} is TEXT" do + test "{:map, _} is TEXT or BLOB" do assert DataType.column_type({:map, %{}}, nil) == "TEXT" + + Application.put_env(:ecto_sqlite3, :map_type, :binary) + + assert DataType.column_type({:map, %{}}, nil) == "BLOB" end test ":array is TEXT" do diff --git a/test/ecto/integration/json_test.exs b/test/ecto/integration/json_test.exs index 823395f..7b350a1 100644 --- a/test/ecto/integration/json_test.exs +++ b/test/ecto/integration/json_test.exs @@ -1,5 +1,5 @@ defmodule Ecto.Integration.JsonTest do - use Ecto.Integration.Case + use Ecto.Integration.Case, async: false alias Ecto.Adapters.SQL alias Ecto.Integration.TestRepo @@ -7,7 +7,34 @@ defmodule Ecto.Integration.JsonTest do @moduletag :integration - test "serializes json correctly" do + setup do + Application.put_env(:ecto_sqlite3, :map_type, :string) + on_exit(fn -> Application.put_env(:ecto_sqlite3, :map_type, :string) end) + end + + test "serializes json correctly with string format" do + # Insert a record purposefully with atoms as the map key. We are going to + # verify later they were coerced into strings. + setting = + %Setting{} + |> Setting.changeset(%{properties: %{foo: "bar", qux: "baz"}}) + |> TestRepo.insert!() + + # Read the record back using ecto and confirm it + assert %Setting{properties: %{"foo" => "bar", "qux" => "baz"}} = + TestRepo.get(Setting, setting.id) + + assert %{num_rows: 1, rows: [["bar"]]} = + SQL.query!( + TestRepo, + "select json_extract(properties, '$.foo') from settings where id = ?1", + [setting.id] + ) + end + + test "serializes json correctly with binary format" do + Application.put_env(:ecto_sqlite3, :map_type, :binary) + # Insert a record purposefully with atoms as the map key. We are going to # verify later they were coerced into strings. setting = From 3b789cd411e6d969493552f5eb10ad887721d715 Mon Sep 17 00:00:00 2001 From: Nassim Date: Wed, 19 Mar 2025 00:08:03 +0100 Subject: [PATCH 2/3] Make array encoding configurable --- lib/ecto/adapters/sqlite3.ex | 3 +++ lib/ecto/adapters/sqlite3/data_type.ex | 16 ++++++++++++++-- test/ecto/adapters/sqlite3/data_type_test.exs | 14 ++++++++++++-- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/lib/ecto/adapters/sqlite3.ex b/lib/ecto/adapters/sqlite3.ex index 9ee9d5b..2e5832d 100644 --- a/lib/ecto/adapters/sqlite3.ex +++ b/lib/ecto/adapters/sqlite3.ex @@ -54,6 +54,9 @@ defmodule Ecto.Adapters.SQLite3 do * `:map_type` - Defaults to `:string`. Determines the type of `:map` columns. Set to `:binary` to use the [JSONB](https://sqlite.org/jsonb.html) storage format. + * `:array_type` - Defaults to `:string`. Determines the type of `:array` columns. + Arrays are serialized using JSON. Set to `:binary` to use the + [JSONB](https://sqlite.org/jsonb.html) storage format. * `:datetime_type` - Defaults to `:iso8601`. Determines how datetime fields are stored in the database. The allowed values are `:iso8601` and `:text_datetime`. `:iso8601` corresponds to a string of the form `YYYY-MM-DDThh:mm:ss` and diff --git a/lib/ecto/adapters/sqlite3/data_type.ex b/lib/ecto/adapters/sqlite3/data_type.ex index d021918..25a614d 100644 --- a/lib/ecto/adapters/sqlite3/data_type.ex +++ b/lib/ecto/adapters/sqlite3/data_type.ex @@ -17,8 +17,6 @@ defmodule Ecto.Adapters.SQLite3.DataType do def column_type(:string, _opts), do: "TEXT" def column_type(:float, _opts), do: "NUMERIC" def column_type(:binary, _opts), do: "BLOB" - def column_type(:array, _opts), do: "TEXT" - def column_type({:array, _}, _opts), do: "TEXT" def column_type(:date, _opts), do: "TEXT" def column_type(:utc_datetime, _opts), do: "TEXT" def column_type(:utc_datetime_usec, _opts), do: "TEXT" @@ -41,6 +39,20 @@ defmodule Ecto.Adapters.SQLite3.DataType do end end + def column_type(:array, _opts) do + case Application.get_env(:ecto_sqlite3, :array_type, :string) do + :string -> "TEXT" + :binary -> "BLOB" + end + end + + def column_type({:array, _}, _opts) do + case Application.get_env(:ecto_sqlite3, :array_type, :string) do + :string -> "TEXT" + :binary -> "BLOB" + end + end + def column_type(:binary_id, _opts) do case Application.get_env(:ecto_sqlite3, :binary_id_type, :string) do :string -> "TEXT" diff --git a/test/ecto/adapters/sqlite3/data_type_test.exs b/test/ecto/adapters/sqlite3/data_type_test.exs index 6ce8979..c0bad2c 100644 --- a/test/ecto/adapters/sqlite3/data_type_test.exs +++ b/test/ecto/adapters/sqlite3/data_type_test.exs @@ -4,11 +4,13 @@ defmodule Ecto.Adapters.SQLite3.DataTypeTest do alias Ecto.Adapters.SQLite3.DataType setup do + Application.put_env(:ecto_sqlite3, :array_type, :string) Application.put_env(:ecto_sqlite3, :binary_id_type, :string) Application.put_env(:ecto_sqlite3, :map_type, :string) Application.put_env(:ecto_sqlite3, :uuid_type, :string) on_exit(fn -> + Application.put_env(:ecto_sqlite3, :array_type, :string) Application.put_env(:ecto_sqlite3, :binary_id_type, :string) Application.put_env(:ecto_sqlite3, :map_type, :string) Application.put_env(:ecto_sqlite3, :uuid_type, :string) @@ -64,12 +66,20 @@ defmodule Ecto.Adapters.SQLite3.DataTypeTest do assert DataType.column_type({:map, %{}}, nil) == "BLOB" end - test ":array is TEXT" do + test ":array is TEXT or BLOB" do assert DataType.column_type(:array, nil) == "TEXT" + + Application.put_env(:ecto_sqlite3, :array_type, :binary) + + assert DataType.column_type(:array, nil) == "BLOB" end - test "{:array, _} is TEXT" do + test "{:array, _} is TEXT or BLOB" do assert DataType.column_type({:array, []}, nil) == "TEXT" + + Application.put_env(:ecto_sqlite3, :array_type, :binary) + + assert DataType.column_type({:array, []}, nil) == "BLOB" end test ":float is NUMERIC" do From ffeaeffdcba8585da3de9f636a46f5ffbed8919c Mon Sep 17 00:00:00 2001 From: Nassim Date: Wed, 19 Mar 2025 00:53:32 +0100 Subject: [PATCH 3/3] Add changelog entry --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79ceda0..79a4ff5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes will be documented in this file. The format is loosely based on [Keep a Changelog][keepachangelog], and this project adheres to [Semantic Versioning][semver]. +## Unreleased +- changed: Configurable encoding for `:map` and `:array`, allowing usage of SQLite's JSONB storage format. + ## v0.18.1 - fixed: Support both `Jason` and `JSON`.