From ff6b2e1b48a426ab6f07141471303192baae51c1 Mon Sep 17 00:00:00 2001 From: spicychickensauce Date: Fri, 18 Jul 2025 14:09:57 +0200 Subject: [PATCH 01/10] Enable tests for values --- integration_test/test_helper.exs | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration_test/test_helper.exs b/integration_test/test_helper.exs index a5dd7b9..1105e4f 100644 --- a/integration_test/test_helper.exs +++ b/integration_test/test_helper.exs @@ -126,9 +126,6 @@ excludes = [ # SQLite does not support anything except a single column in DISTINCT :multicolumn_distinct, - - # Values list - :values_list ] ExUnit.configure(exclude: excludes) From bbd1dbb095fb99738cd118f0f70d01ac656fdc4e Mon Sep 17 00:00:00 2001 From: spicychickensauce Date: Fri, 18 Jul 2025 14:15:18 +0200 Subject: [PATCH 02/10] Implement `values` via CTE --- lib/ecto/adapters/sqlite3/connection.ex | 43 ++++++++++++++++++++----- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/lib/ecto/adapters/sqlite3/connection.ex b/lib/ecto/adapters/sqlite3/connection.ex index fcdd12c..554fc82 100644 --- a/lib/ecto/adapters/sqlite3/connection.ex +++ b/lib/ecto/adapters/sqlite3/connection.ex @@ -1037,12 +1037,6 @@ defmodule Ecto.Adapters.SQLite3.Connection do message: "join hints are not supported by SQLite3" end - defp assert_valid_join(%JoinExpr{source: {:values, _, _}}, query) do - raise Ecto.QueryError, - query: query, - message: "SQLite3 adapter does not support values lists" - end - defp assert_valid_join(_join_expr, _query), do: :ok defp join_on(:cross, true, _sources, _query), do: [] @@ -1368,8 +1362,8 @@ defmodule Ecto.Adapters.SQLite3.Connection do |> parens_for_select end - defp expr({:values, _, _}, _, _query) do - raise ArgumentError, "SQLite3 adapter does not support values lists" + defp expr({:values, _, [types, idx, num_rows]}, _, _query) do + [?(, values_list(types, idx + 1, num_rows), ?)] end defp expr({:identifier, _, [literal]}, _sources, _query) do @@ -1560,6 +1554,36 @@ defmodule Ecto.Adapters.SQLite3.Connection do message: "unsupported expression #{inspect(expr)}" end + defp values_list(types, idx, num_rows) do + rows = :lists.seq(1, num_rows, 1) + col_names = Enum.map_join(types, ", ", &elem(&1, 0)) + + [ + "WITH xxx(", + col_names, + ") AS (VALUES ", + intersperse_reduce(rows, ?,, idx, fn _, idx -> + {value, idx} = values_expr(types, idx) + {[?(, value, ?)], idx} + end) + |> elem(0), + ") SELECT * FROM xxx" + ] + end + + defp values_expr(types, idx) do + intersperse_reduce(types, ?,, idx, fn {_field, _type}, idx -> + # TODO: cast? + # {[?$, Integer.to_string(idx), ?:, ?: | tagged_to_db(type)], idx + 1} + {[?$, Integer.to_string(idx)], idx + 1} + end) + end + + # defp tagged_to_db(:id), do: "bigint" + # defp tagged_to_db(:integer), do: "bigint" + # defp tagged_to_db({:array, type}), do: [tagged_to_db(type), ?[, ?]] + # defp tagged_to_db(type), do: ecto_to_db(type) + def interval(_, "microsecond", _sources) do raise ArgumentError, "SQLite does not support microsecond precision in datetime intervals" @@ -1618,6 +1642,9 @@ defmodule Ecto.Adapters.SQLite3.Connection do {:fragment, _, _} -> {nil, as_prefix ++ [?f | Integer.to_string(pos)], nil} + {:values, _, _} -> + {nil, as_prefix ++ [?v | Integer.to_string(pos)], nil} + {table, schema, prefix} -> name = as_prefix ++ [create_alias(table) | Integer.to_string(pos)] {quote_table(prefix, table), name, schema} From cafc720cbce5676af1c69db2d1d88cda9e3863f3 Mon Sep 17 00:00:00 2001 From: spicychickensauce Date: Fri, 18 Jul 2025 14:46:14 +0200 Subject: [PATCH 03/10] Add snapshot test of join with values query --- .../adapters/sqlite3/connection/join_test.exs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/test/ecto/adapters/sqlite3/connection/join_test.exs b/test/ecto/adapters/sqlite3/connection/join_test.exs index fbc0409..aba4264 100644 --- a/test/ecto/adapters/sqlite3/connection/join_test.exs +++ b/test/ecto/adapters/sqlite3/connection/join_test.exs @@ -135,11 +135,11 @@ defmodule Ecto.Adapters.SQLite3.Connection.JoinTest do end end - test "join with values is not supported" do - assert_raise Ecto.QueryError, fn -> - rows = [%{x: 1, y: 1}, %{x: 2, y: 2}] - types = %{x: :integer, y: :integer} + test "join with values" do + rows = [%{x: 1, y: 1}, %{x: 2, y: 2}] + types = %{x: :integer, y: :integer} + query = Schema |> join( :inner, @@ -149,8 +149,11 @@ defmodule Ecto.Adapters.SQLite3.Connection.JoinTest do ) |> select([p, q], {p.id, q.x}) |> plan() - |> all() - end + + assert ~s{SELECT s0."id", v1."x" FROM "schema" AS s0 } <> + ~s{INNER JOIN (WITH xxx(y, x) AS (VALUES ($1,$2),($3,$4)) SELECT * FROM xxx) AS v1 } <> + ~s{ON (v1."x" = s0."x") AND (v1."y" = s0."y")} == + all(query) end test "join with fragment" do From 40733b8ee94739b46d4f74a7840eca288ef10a8d Mon Sep 17 00:00:00 2001 From: spicychickensauce Date: Fri, 18 Jul 2025 16:20:16 +0200 Subject: [PATCH 04/10] Test `values` when combined with `with_cte` --- integration_test/values_test.exs | 59 ++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 integration_test/values_test.exs diff --git a/integration_test/values_test.exs b/integration_test/values_test.exs new file mode 100644 index 0000000..e46d09c --- /dev/null +++ b/integration_test/values_test.exs @@ -0,0 +1,59 @@ +defmodule Ecto.Integration.ValuesTest do + use Ecto.Integration.Case, async: true + + import Ecto.Query, only: [from: 2, with_cte: 3] + + alias Ecto.Integration.Comment + alias Ecto.Integration.Post + alias Ecto.Integration.TestRepo + + test "join to values works" do + TestRepo.insert!(%Post{id: 1}) + TestRepo.insert!(%Comment{post_id: 1, text: "short"}) + TestRepo.insert!(%Comment{post_id: 1, text: "much longer text"}) + + params = [%{id: 1, post_id: 1, n: 0}, %{id: 2, post_id: 1, n: 10}] + types = %{id: :integer, post_id: :integer, n: :integer} + + results = + from(p in Post, + right_join: params in values(params, types), + on: params.post_id == p.id, + left_join: c in Comment, + on: c.post_id == p.id and fragment("LENGTH(?)", c.text) > params.n, + group_by: params.id, + select: {params.id, count(c.id)} + ) + |> TestRepo.all() + + assert [{1, 2}, {2, 1}] = results + end + + test "values can be used together with CTE" do + TestRepo.insert!(%Post{id: 1, visits: 42}) + TestRepo.insert!(%Comment{post_id: 1, text: "short"}) + TestRepo.insert!(%Comment{post_id: 1, text: "much longer text"}) + + params = [%{id: 1, post_id: 1, n: 0}, %{id: 2, post_id: 1, n: 10}] + types = %{id: :integer, post_id: :integer, n: :integer} + + cte_query = from(p in Post, select: %{id: p.id, visits: coalesce(p.visits, 0)}) + + q = Post |> with_cte("xxx", as: ^cte_query) + + results = + from(p in q, + right_join: params in values(params, types), + on: params.post_id == p.id, + left_join: c in Comment, + on: c.post_id == p.id and fragment("LENGTH(?)", c.text) > params.n, + left_join: cte in "xxx", + on: cte.id == p.id, + group_by: params.id, + select: {params.id, count(c.id), cte.visits} + ) + |> TestRepo.all() + + assert [{1, 2, 42}, {2, 1, 42}] = results + end +end From 2b01d456c30cbcb511fb12f05466ee3d93e1afde Mon Sep 17 00:00:00 2001 From: spicychickensauce Date: Sat, 19 Jul 2025 14:41:19 +0200 Subject: [PATCH 05/10] Add test for values using naive_datetime --- integration_test/values_test.exs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/integration_test/values_test.exs b/integration_test/values_test.exs index e46d09c..9483908 100644 --- a/integration_test/values_test.exs +++ b/integration_test/values_test.exs @@ -7,6 +7,38 @@ defmodule Ecto.Integration.ValuesTest do alias Ecto.Integration.Post alias Ecto.Integration.TestRepo + test "values works with datetime" do + TestRepo.insert!(%Post{inserted_at: ~N[2000-01-01 00:01:00]}) + TestRepo.insert!(%Post{inserted_at: ~N[2000-01-01 00:02:00]}) + TestRepo.insert!(%Post{inserted_at: ~N[2000-01-01 00:03:00]}) + + params = [ + %{id: 1, date: ~N[2000-01-01 00:00:00]}, + %{id: 2, date: ~N[2000-01-01 00:01:00]}, + %{id: 3, date: ~N[2000-01-01 00:02:00]}, + %{id: 4, date: ~N[2000-01-01 00:03:00]} + ] + + types = %{id: :integer, date: :naive_datetime} + + results = + from(params in values(params, types), + left_join: p in Post, + on: p.inserted_at <= params.date, + group_by: params.id, + select: %{id: params.id, count: count(p.id)}, + order_by: count(p.id) + ) + |> TestRepo.all() + + assert results == [ + %{count: 0, id: 1}, + %{count: 1, id: 2}, + %{count: 2, id: 3}, + %{count: 3, id: 4} + ] + end + test "join to values works" do TestRepo.insert!(%Post{id: 1}) TestRepo.insert!(%Comment{post_id: 1, text: "short"}) From e8516847d866184100ee3acabbd3ef0161313ba3 Mon Sep 17 00:00:00 2001 From: spicychickensauce Date: Sat, 19 Jul 2025 15:01:03 +0200 Subject: [PATCH 06/10] Use random name for temp table --- lib/ecto/adapters/sqlite3/connection.ex | 5 +++-- test/ecto/adapters/sqlite3/connection/join_test.exs | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/ecto/adapters/sqlite3/connection.ex b/lib/ecto/adapters/sqlite3/connection.ex index 554fc82..d1b4598 100644 --- a/lib/ecto/adapters/sqlite3/connection.ex +++ b/lib/ecto/adapters/sqlite3/connection.ex @@ -1557,9 +1557,10 @@ defmodule Ecto.Adapters.SQLite3.Connection do defp values_list(types, idx, num_rows) do rows = :lists.seq(1, num_rows, 1) col_names = Enum.map_join(types, ", ", &elem(&1, 0)) + table_name = "temp_#{:rand.uniform(100_000)}" [ - "WITH xxx(", + "WITH #{table_name}(", col_names, ") AS (VALUES ", intersperse_reduce(rows, ?,, idx, fn _, idx -> @@ -1567,7 +1568,7 @@ defmodule Ecto.Adapters.SQLite3.Connection do {[?(, value, ?)], idx} end) |> elem(0), - ") SELECT * FROM xxx" + ") SELECT * FROM #{table_name}" ] end diff --git a/test/ecto/adapters/sqlite3/connection/join_test.exs b/test/ecto/adapters/sqlite3/connection/join_test.exs index aba4264..bd089bb 100644 --- a/test/ecto/adapters/sqlite3/connection/join_test.exs +++ b/test/ecto/adapters/sqlite3/connection/join_test.exs @@ -139,6 +139,9 @@ defmodule Ecto.Adapters.SQLite3.Connection.JoinTest do rows = [%{x: 1, y: 1}, %{x: 2, y: 2}] types = %{x: :integer, y: :integer} + # Seeding rand ensures we get temp_78027 as the CTE name + :rand.seed(:exsss, {1, 2, 3}) + query = Schema |> join( @@ -151,7 +154,7 @@ defmodule Ecto.Adapters.SQLite3.Connection.JoinTest do |> plan() assert ~s{SELECT s0."id", v1."x" FROM "schema" AS s0 } <> - ~s{INNER JOIN (WITH xxx(y, x) AS (VALUES ($1,$2),($3,$4)) SELECT * FROM xxx) AS v1 } <> + ~s{INNER JOIN (WITH temp_78027(y, x) AS (VALUES ($1,$2),($3,$4)) SELECT * FROM temp_78027) AS v1 } <> ~s{ON (v1."x" = s0."x") AND (v1."y" = s0."y")} == all(query) end From 2054a79fb6a95eb401889803134c3a28b2d587a8 Mon Sep 17 00:00:00 2001 From: spicychickensauce Date: Sat, 19 Jul 2025 15:32:41 +0200 Subject: [PATCH 07/10] Exclude out of scope values_list test --- integration_test/test_helper.exs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration_test/test_helper.exs b/integration_test/test_helper.exs index 1105e4f..fef846a 100644 --- a/integration_test/test_helper.exs +++ b/integration_test/test_helper.exs @@ -126,6 +126,10 @@ excludes = [ # SQLite does not support anything except a single column in DISTINCT :multicolumn_distinct, + + # Run all with tag values_list, except for the "delete_all" test, + # as JOINS are not supported on DELETE statements by SQLite. + {:location, {"ecto/integration_test/cases/repo.exs", 2281}} ] ExUnit.configure(exclude: excludes) From 3183b70d49f671ab8a62c406c86776579c686f07 Mon Sep 17 00:00:00 2001 From: spicychickensauce Date: Sat, 19 Jul 2025 15:56:07 +0200 Subject: [PATCH 08/10] Ensure test works across multiple OTP versions --- test/ecto/adapters/sqlite3/connection/join_test.exs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/ecto/adapters/sqlite3/connection/join_test.exs b/test/ecto/adapters/sqlite3/connection/join_test.exs index bd089bb..ec932de 100644 --- a/test/ecto/adapters/sqlite3/connection/join_test.exs +++ b/test/ecto/adapters/sqlite3/connection/join_test.exs @@ -153,8 +153,12 @@ defmodule Ecto.Adapters.SQLite3.Connection.JoinTest do |> select([p, q], {p.id, q.x}) |> plan() + # We can't hardcode the columns of the CTE here, + # as the order changes for different Map implementations for OTP 25 and later + col_names = Map.keys(types) |> Enum.join(", ") + assert ~s{SELECT s0."id", v1."x" FROM "schema" AS s0 } <> - ~s{INNER JOIN (WITH temp_78027(y, x) AS (VALUES ($1,$2),($3,$4)) SELECT * FROM temp_78027) AS v1 } <> + ~s{INNER JOIN (WITH temp_78027(#{col_names}) AS (VALUES ($1,$2),($3,$4)) SELECT * FROM temp_78027) AS v1 } <> ~s{ON (v1."x" = s0."x") AND (v1."y" = s0."y")} == all(query) end From b86ecbb1ff269d314ea7d64a2ef915341bcf841b Mon Sep 17 00:00:00 2001 From: spicychickensauce Date: Tue, 22 Jul 2025 10:38:58 +0200 Subject: [PATCH 09/10] Do not run :values_list tests on elixir 1.15 --- integration_test/test_helper.exs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/integration_test/test_helper.exs b/integration_test/test_helper.exs index fef846a..64ff447 100644 --- a/integration_test/test_helper.exs +++ b/integration_test/test_helper.exs @@ -127,9 +127,14 @@ excludes = [ # SQLite does not support anything except a single column in DISTINCT :multicolumn_distinct, - # Run all with tag values_list, except for the "delete_all" test, - # as JOINS are not supported on DELETE statements by SQLite. - {:location, {"ecto/integration_test/cases/repo.exs", 2281}} + # :location is not supported in elixir 1.15 and earlier, so we exclude all + if Version.match?(System.version(), "~> 1.16") do + # Run all with tag values_list, except for the "delete_all" test, + # as JOINS are not supported on DELETE statements by SQLite. + {:location, {"ecto/integration_test/cases/repo.exs", 2281}} + else + :values_list + end ] ExUnit.configure(exclude: excludes) From 99cec9a53334e6f3b35dcb4b317b4a5dc0876f5a Mon Sep 17 00:00:00 2001 From: spicychickensauce Date: Tue, 22 Jul 2025 10:53:55 +0200 Subject: [PATCH 10/10] Cast to appropriate type in values expression --- lib/ecto/adapters/sqlite3/connection.ex | 11 ++--------- test/ecto/adapters/sqlite3/connection/join_test.exs | 3 ++- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/lib/ecto/adapters/sqlite3/connection.ex b/lib/ecto/adapters/sqlite3/connection.ex index d1b4598..516f83e 100644 --- a/lib/ecto/adapters/sqlite3/connection.ex +++ b/lib/ecto/adapters/sqlite3/connection.ex @@ -1573,18 +1573,11 @@ defmodule Ecto.Adapters.SQLite3.Connection do end defp values_expr(types, idx) do - intersperse_reduce(types, ?,, idx, fn {_field, _type}, idx -> - # TODO: cast? - # {[?$, Integer.to_string(idx), ?:, ?: | tagged_to_db(type)], idx + 1} - {[?$, Integer.to_string(idx)], idx + 1} + intersperse_reduce(types, ?,, idx, fn {_field, type}, idx -> + {[?$, Integer.to_string(idx), ?:, ?: | column_type(type, nil)], idx + 1} end) end - # defp tagged_to_db(:id), do: "bigint" - # defp tagged_to_db(:integer), do: "bigint" - # defp tagged_to_db({:array, type}), do: [tagged_to_db(type), ?[, ?]] - # defp tagged_to_db(type), do: ecto_to_db(type) - def interval(_, "microsecond", _sources) do raise ArgumentError, "SQLite does not support microsecond precision in datetime intervals" diff --git a/test/ecto/adapters/sqlite3/connection/join_test.exs b/test/ecto/adapters/sqlite3/connection/join_test.exs index ec932de..749c786 100644 --- a/test/ecto/adapters/sqlite3/connection/join_test.exs +++ b/test/ecto/adapters/sqlite3/connection/join_test.exs @@ -158,7 +158,8 @@ defmodule Ecto.Adapters.SQLite3.Connection.JoinTest do col_names = Map.keys(types) |> Enum.join(", ") assert ~s{SELECT s0."id", v1."x" FROM "schema" AS s0 } <> - ~s{INNER JOIN (WITH temp_78027(#{col_names}) AS (VALUES ($1,$2),($3,$4)) SELECT * FROM temp_78027) AS v1 } <> + ~s{INNER JOIN (WITH temp_78027(#{col_names}) AS (VALUES ($1::INTEGER,$2::INTEGER),($3::INTEGER,$4::INTEGER)) } <> + ~s{SELECT * FROM temp_78027) AS v1 } <> ~s{ON (v1."x" = s0."x") AND (v1."y" = s0."y")} == all(query) end