Refactor benchmarks

Author: Schultzer

bench.exs renamed to benchmarks/bench.exs

@@ -15,7 +15,7 @@ defmodule SQL.Repo do
   end
 end
 Application.put_env(:sql, :ecto_repos, [SQL.Repo])
-Application.put_env(:sql, SQL.Repo, log: false, username: "postgres", password: "postgres", hostname: "localhost", database: "sql_test#{System.get_env("MIX_TEST_PARTITION")}", pool: Ecto.Adapters.SQL.Sandbox, pool_size: 10)
+Application.put_env(:sql, SQL.Repo, log: false, username: "postgres", password: "postgres", hostname: "localhost", database: "sql_test#{System.get_env("MIX_TEST_PARTITION")}", pool_size: 10)
 SQL.Repo.__adapter__().storage_up(SQL.Repo.config())
 SQL.Repo.start_link()
 query = "temp" |> recursive_ctes(true) |> with_cte("temp", as: ^union_all(select("temp", [t], %{n: 0, fact: 1}), ^where(select("temp", [t], [t.n+1, t.n+1*t.fact]), [t], t.n < 9))) |> select([t], [t.n])
@@ -32,13 +32,13 @@ Benchee.run(
   "lex" => fn _ -> SQL.Lexer.lex("with recursive temp (n, fact) as (select 0, 1 union all select n+1, (n+1)*fact from temp where n < 9)") end,
   "parse" => fn _ -> SQL.Parser.parse(tokens, context) end,
   "iodata" => fn _ -> pcontext.module.to_iodata(ptokens, pcontext)  end,
-  "format" => fn _ -> SQL.Format.to_iodata(ptokens, pcontext)  end,
+  "format" => fn _ -> SQL.Format.to_iodata(ptokens, pcontext, 0, false)  end,
   "lex+parse+iodata" => fn _ ->
     {:ok, _, tokens} = SQL.Lexer.lex("with recursive temp (n, fact) as (select 0, 1 union all select n+1, (n+1)*fact from temp where n < 9)")
     {:ok, pcontext, tokens} = SQL.Parser.parse(tokens, context)
     pcontext.module.to_iodata(tokens, pcontext)
   end,
-  "parse/1" => fn _ -> SQL.parse("with recursive temp (n, fact) as (select 0, 1 union all select n+1, (n+1)*fact from temp where n < 9)") end,
+  "parse/3" => fn _ -> SQL.parse("with recursive temp (n, fact) as (select 0, 1 union all select n+1, (n+1)*fact from temp where n < 9)") end,
   "sql" => fn _ -> SQL.Repo.sql() end,
   "ecto" => fn _ -> SQL.Repo.ecto() end,
   "runtime to_string" => fn _ -> to_string(~SQL[with recursive temp (n, fact) as (select 0, 1 union all select n+1, (n+1)*fact from temp where n < 9)]) end,

benchmarks/pool.exs

@@ -0,0 +1,33 @@
+import SQL
+import Ecto.Query
+SQL.Pool.start_link(%{name: :mypool, protocol: :tcp, size: 10})
+defmodule SQL.Repo do
+  use Ecto.Repo, otp_app: :sql, adapter: Ecto.Adapters.Postgres
+  use SQL, adapter: SQL.Adapters.Postgres, repo: __MODULE__
+
+  def sql() do
+    {idx, _} = SQL.Pool.checkout(SQL.parse("with recursive temp (n, fact) as (select 0, 1 union all select n+1, (n+1)*fact from temp where n < 9)"), :mypool)
+    SQL.Pool.checkin(idx, :mypool)
+  end
+
+  def ecto() do
+    checkout(fn -> SQL.parse("with recursive temp (n, fact) as (select 0, 1 union all select n+1, (n+1)*fact from temp where n < 9)") end)
+  end
+end
+Application.put_env(:sql, :ecto_repos, [SQL.Repo])
+Application.put_env(:sql, SQL.Repo, log: false, username: "postgres", password: "postgres", hostname: "localhost", database: "sql_test#{System.get_env("MIX_TEST_PARTITION")}", pool_size: 10)
+SQL.Repo.__adapter__().storage_up(SQL.Repo.config())
+SQL.Repo.start_link()
+Benchee.run(
+  %{
+  "sql" => fn _ -> SQL.Repo.sql() end,
+  "ecto" => fn _ -> SQL.Repo.ecto() end,
+  },
+  parallel: 50, time: 1,
+  inputs: %{"1..100_000" => Enum.to_list(1..100_000)},
+  memory_time: 2,
+  reduction_time: 2,
+  unit_scaling: :smallest,
+  measure_function_call_overhead: true,
+  profile_after: :eprof
+)

benchmarks/v0.4.0.md

@@ -0,0 +1,78 @@
+➜  sql git:(main) ✗ mix sql.bench
+Operating System: macOS
+CPU Information: Apple M1 Max
+Number of Available Cores: 10
+Available memory: 64 GB
+Elixir 1.20.0-dev
+Erlang 28.1
+JIT enabled: true
+
+Benchmark suite executing with the following configuration:
+warmup: 2 s
+time: 5 s
+memory time: 2 s
+reduction time: 2 s
+parallel: 1
+inputs: 1..100_000
+Estimated total run time: 1 min 28 s
+
+Measured function call overhead as: 0 ns
+Benchmarking comptime ecto with input 1..100_000 ...
+Benchmarking format with input 1..100_000 ...
+Benchmarking iodata with input 1..100_000 ...
+Benchmarking lex with input 1..100_000 ...
+Benchmarking lex+parse+iodata with input 1..100_000 ...
+Benchmarking parse with input 1..100_000 ...
+Benchmarking parse/3 with input 1..100_000 ...
+Benchmarking runtime ecto with input 1..100_000 ...
+Calculating statistics...
+Formatting results...
+
+##### With input 1..100_000 #####
+Name                       ips        average  deviation         median         99th %
+iodata               1627.29 K      614.52 ns  ±3180.45%         542 ns         709 ns
+format               1296.69 K      771.20 ns  ±2729.73%         708 ns         917 ns
+parse                1049.15 K      953.15 ns  ±1490.08%         875 ns        1083 ns
+lex                   830.98 K     1203.40 ns  ±1480.24%        1084 ns        1584 ns
+lex+parse+iodata      383.17 K     2609.80 ns   ±478.57%        2500 ns        3125 ns
+parse/3               308.89 K     3237.35 ns   ±316.26%        3000 ns        8125 ns
+comptime ecto         199.44 K     5014.02 ns   ±156.64%        4791 ns        8958 ns
+runtime ecto          185.75 K     5383.47 ns   ±169.39%        5084 ns    13579.31 ns
+
+Comparison:
+iodata               1627.29 K
+format               1296.69 K - 1.25x slower +156.68 ns
+parse                1049.15 K - 1.55x slower +338.63 ns
+lex                   830.98 K - 1.96x slower +588.88 ns
+lex+parse+iodata      383.17 K - 4.25x slower +1995.29 ns
+parse/3               308.89 K - 5.27x slower +2622.83 ns
+comptime ecto         199.44 K - 8.16x slower +4399.50 ns
+runtime ecto          185.75 K - 8.76x slower +4768.95 ns
+
+Memory usage statistics:
+
+Name                Memory usage
+iodata                   1.35 KB
+format                   1.42 KB - 1.05x memory usage +0.0703 KB
+parse                    2.23 KB - 1.65x memory usage +0.88 KB
+lex                      8.63 KB - 6.39x memory usage +7.28 KB
+lex+parse+iodata        12.21 KB - 9.03x memory usage +10.86 KB
+parse/3                 12.73 KB - 9.42x memory usage +11.38 KB
+comptime ecto           17.78 KB - 13.16x memory usage +16.43 KB
+runtime ecto            20.21 KB - 14.95x memory usage +18.86 KB
+
+**All measurements for memory usage were the same**
+
+Reduction count statistics:
+
+Name             Reduction count
+iodata                       175
+format                       188 - 1.07x reduction count +13
+parse                        194 - 1.11x reduction count +19
+lex                          345 - 1.97x reduction count +170
+lex+parse+iodata             714 - 4.08x reduction count +539
+parse/3                      763 - 4.36x reduction count +588
+comptime ecto               1132 - 6.47x reduction count +957
+runtime ecto                1193 - 6.82x reduction count +1018
+
+**All measurements for reduction count were the same**

mix.exs

@@ -16,7 +16,7 @@ defmodule SQL.MixProject do
       name: "SQL",
       docs: docs(),
       package: package(),
-      aliases: ["sql.bench": "run bench.exs"]
+      aliases: ["sql.bench": "run benchmarks/bench.exs", "sql.bench.pool": "run benchmarks/pool.exs"]
     ]
   end