diff --git a/lib/philomena/comments/query.ex b/lib/philomena/comments/query.ex index 49ec6d688..4954e1c32 100644 --- a/lib/philomena/comments/query.ex +++ b/lib/philomena/comments/query.ex @@ -51,8 +51,8 @@ defmodule Philomena.Comments.Query do defp anonymous_fields do [ int_fields: ~W(id), + numeric_fields: ~W(image_id), date_fields: ~W(created_at), - literal_fields: ~W(image_id), ngram_fields: ~W(body), custom_fields: ~W(author user_id), default_field: {"body", :ngram}, @@ -77,7 +77,8 @@ defmodule Philomena.Comments.Query do fields = user_fields() Keyword.merge(fields, - literal_fields: ~W(image_id user_id author fingerprint), + numeric_fields: fields[:numeric_fields] ++ ~W(user_id), + literal_fields: ~W(author fingerprint), ip_fields: ~W(ip), bool_fields: ~W(anonymous deleted), custom_fields: fields[:custom_fields] -- ~W(author user_id), diff --git a/lib/philomena/filters/query.ex b/lib/philomena/filters/query.ex index 3460a459c..dd3512f04 100644 --- a/lib/philomena/filters/query.ex +++ b/lib/philomena/filters/query.ex @@ -10,9 +10,10 @@ defmodule Philomena.Filters.Query do defp anonymous_fields do [ int_fields: ~W(id spoilered_count hidden_count), + numeric_fields: ~W(user_id), date_fields: ~W(created_at), ngram_fields: ~W(description), - literal_fields: ~W(name creator user_id), + literal_fields: ~W(name creator), bool_fields: ~W(public system), default_field: {"name", :term} ] diff --git a/lib/philomena/galleries/query.ex b/lib/philomena/galleries/query.ex index ddfa1e8bc..26e99f4ba 100644 --- a/lib/philomena/galleries/query.ex +++ b/lib/philomena/galleries/query.ex @@ -4,7 +4,8 @@ defmodule Philomena.Galleries.Query do defp fields do [ int_fields: ~W(id image_count watcher_count), - literal_fields: ~W(title user image_ids watcher_ids), + numeric_fields: ~W(image_ids watcher_ids), + literal_fields: ~W(title user), date_fields: ~W(created_at updated_at), ngram_fields: ~W(description), default_field: {"title", :term}, diff --git a/lib/philomena/images/query.ex b/lib/philomena/images/query.ex index 901147ac1..b84df6219 100644 --- a/lib/philomena/images/query.ex +++ b/lib/philomena/images/query.ex @@ -85,8 +85,9 @@ defmodule Philomena.Images.Query do defp anonymous_fields do [ int_fields: - ~W(id width height score upvotes downvotes faves uploader_id faved_by_id pixels size orig_size comment_count source_count tag_count) ++ + ~W(id width height score upvotes downvotes faves pixels size orig_size comment_count source_count tag_count) ++ tag_count_fields(), + numeric_fields: ~W(uploader_id faved_by_id duplicate_id), float_fields: ~W(aspect_ratio wilson_score duration), date_fields: ~W(created_at updated_at first_seen_at), literal_fields: @@ -118,8 +119,8 @@ defmodule Philomena.Images.Query do fields = user_fields() Keyword.merge(fields, - int_fields: - fields[:int_fields] ++ + numeric_fields: + fields[:numeric_fields] ++ ~W(upvoted_by_id downvoted_by_id true_uploader_id hidden_by_id deleted_by_user_id), literal_fields: fields[:literal_fields] ++ diff --git a/lib/philomena/posts/query.ex b/lib/philomena/posts/query.ex index 58d94d6ca..f4836ad13 100644 --- a/lib/philomena/posts/query.ex +++ b/lib/philomena/posts/query.ex @@ -51,8 +51,9 @@ defmodule Philomena.Posts.Query do defp anonymous_fields do [ int_fields: ~W(id topic_position), + numeric_fields: ~W(forum_id topic_id), date_fields: ~W(created_at updated_at), - literal_fields: ~W(forum forum_id topic_id), + literal_fields: ~W(forum), ngram_fields: ~W(body subject), custom_fields: ~W(author user_id), default_field: {"body", :ngram}, @@ -76,7 +77,8 @@ defmodule Philomena.Posts.Query do fields = user_fields() Keyword.merge(fields, - literal_fields: fields[:literal_fields] ++ ~W(user_id author fingerprint), + numeric_fields: fields[:numeric_fields] ++ ~W(user_id), + literal_fields: fields[:literal_fields] ++ ~W(author fingerprint), ip_fields: ~W(ip), bool_fields: ~W(anonymous deleted), custom_fields: fields[:custom_fields] -- ~W(author user_id), diff --git a/lib/philomena/reports/query.ex b/lib/philomena/reports/query.ex index ebff3bf9d..7d45429a5 100644 --- a/lib/philomena/reports/query.ex +++ b/lib/philomena/reports/query.ex @@ -3,10 +3,10 @@ defmodule Philomena.Reports.Query do defp fields do [ - int_fields: ~W(id image_id), + int_fields: ~W(id), + numeric_fields: ~W(user_id admin_id reportable_id image_id), date_fields: ~W(created_at), - literal_fields: - ~W(state user user_id admin admin_id reportable_type reportable_id fingerprint), + literal_fields: ~W(state user admin reportable_type fingerprint), ip_fields: ~W(ip), bool_fields: ~W(open), ngram_fields: ~W(reason), diff --git a/lib/philomena_query/parse/float_parser.ex b/lib/philomena_query/parse/float_parser.ex index 2fa253fc4..c9773e6f8 100644 --- a/lib/philomena_query/parse/float_parser.ex +++ b/lib/philomena_query/parse/float_parser.ex @@ -3,9 +3,6 @@ defmodule PhilomenaQuery.Parse.FloatParser do import NimbleParsec - defp to_number(input), do: PhilomenaQuery.Parse.Helpers.to_number(input) - defp range(input), do: PhilomenaQuery.Parse.Helpers.range(input) - space = choice([string(" "), string("\t"), string("\n"), string("\r"), string("\v"), string("\f")]) |> ignore() @@ -18,21 +15,21 @@ defmodule PhilomenaQuery.Parse.FloatParser do ascii_string([?0..?9], min: 1) |> optional(ascii_char(~c".") |> ascii_string([?0..?9], min: 1)) |> reduce({List, :to_string, []}) - |> reduce(:to_number) + |> reduce({PhilomenaQuery.Parse.Helpers, :to_number, []}) float = optional(ascii_char(~c"-+")) |> ascii_string([?0..?9], min: 1) |> optional(ascii_char(~c".") |> ascii_string([?0..?9], min: 1)) |> reduce({List, :to_string, []}) - |> reduce(:to_number) + |> reduce({PhilomenaQuery.Parse.Helpers, :to_number, []}) float_parser = choice([ float |> concat(fuzz) |> concat(unsigned_float) - |> reduce(:range) + |> reduce({PhilomenaQuery.Parse.Helpers, :range, []}) |> unwrap_and_tag(:float_range), float |> unwrap_and_tag(:float) ]) diff --git a/lib/philomena_query/parse/helpers.ex b/lib/philomena_query/parse/helpers.ex index 5d9718900..a0c2483bf 100644 --- a/lib/philomena_query/parse/helpers.ex +++ b/lib/philomena_query/parse/helpers.ex @@ -1,6 +1,9 @@ defmodule PhilomenaQuery.Parse.Helpers do @moduledoc false + @min_int32 -2_147_483_648 + @max_int32 2_147_483_647 + # Apparently, it's too hard for the standard library to to parse a number # as a float if it doesn't contain a decimal point. WTF def to_number(term) do @@ -17,15 +20,18 @@ defmodule PhilomenaQuery.Parse.Helpers do end def to_int(term) do - {int, _} = :string.to_integer(term) + {int, []} = :string.to_integer(term) - if int in -2_147_483_648..2_147_483_647 do - int - else - 0 - end + clamp_to_int32(int) end + def int_range([center, deviation]) do + [clamp_to_int32(center - deviation), clamp_to_int32(center + deviation)] + end + + def clamp_to_int32(int) when is_integer(int), + do: min(max(int, @min_int32), @max_int32) + def range([center, deviation]) do [center - deviation, center + deviation] end diff --git a/lib/philomena_query/parse/int_parser.ex b/lib/philomena_query/parse/int_parser.ex index a3f7842aa..d360df516 100644 --- a/lib/philomena_query/parse/int_parser.ex +++ b/lib/philomena_query/parse/int_parser.ex @@ -3,9 +3,6 @@ defmodule PhilomenaQuery.Parse.IntParser do import NimbleParsec - defp to_int(input), do: PhilomenaQuery.Parse.Helpers.to_int(input) - defp range(input), do: PhilomenaQuery.Parse.Helpers.range(input) - space = choice([string(" "), string("\t"), string("\n"), string("\r"), string("\v"), string("\f")]) |> ignore() @@ -18,16 +15,20 @@ defmodule PhilomenaQuery.Parse.IntParser do optional(ascii_char(~c"-+")) |> ascii_string([?0..?9], min: 1) |> reduce({List, :to_string, []}) - |> reduce(:to_int) + |> reduce({PhilomenaQuery.Parse.Helpers, :to_int, []}) int_parser = choice([ - int |> concat(fuzz) |> integer(min: 1) |> reduce(:range) |> unwrap_and_tag(:int_range), + int + |> concat(fuzz) + |> integer(min: 1) + |> reduce({PhilomenaQuery.Parse.Helpers, :int_range, []}) + |> unwrap_and_tag(:int_range), int |> unwrap_and_tag(:int) ]) |> repeat(space) |> eos() - |> label("an integer, like `3' or `-10'") + |> label("a signed integer, like `3' or `-10'") defparsec(:parse, int_parser) end diff --git a/lib/philomena_query/parse/lexer.ex b/lib/philomena_query/parse/lexer.ex index 648099a4c..422554a5d 100644 --- a/lib/philomena_query/parse/lexer.ex +++ b/lib/philomena_query/parse/lexer.ex @@ -3,8 +3,6 @@ defmodule PhilomenaQuery.Parse.Lexer do import NimbleParsec - defp to_number(input), do: PhilomenaQuery.Parse.Helpers.to_number(input) - space = choice([string(" "), string("\t"), string("\n"), string("\r"), string("\v"), string("\f")]) |> ignore() @@ -14,7 +12,7 @@ defmodule PhilomenaQuery.Parse.Lexer do |> ascii_string([?0..?9], min: 1) |> optional(ascii_char(~c".") |> ascii_string([?0..?9], min: 1)) |> reduce({List, :to_string, []}) - |> reduce(:to_number) + |> reduce({PhilomenaQuery.Parse.Helpers, :to_number, []}) l_and = times(space, min: 1) diff --git a/lib/philomena_query/parse/literal_parser.ex b/lib/philomena_query/parse/literal_parser.ex index 253d647ff..c424bedf2 100644 --- a/lib/philomena_query/parse/literal_parser.ex +++ b/lib/philomena_query/parse/literal_parser.ex @@ -4,13 +4,11 @@ defmodule PhilomenaQuery.Parse.LiteralParser do import NimbleParsec @dialyzer [:no_match, :no_unused] - defp to_number(input), do: PhilomenaQuery.Parse.Helpers.to_number(input) - float = ascii_string([?0..?9], min: 1) |> optional(ascii_char(~c".") |> ascii_string([?0..?9], min: 1)) |> reduce({List, :to_string, []}) - |> reduce(:to_number) + |> reduce({PhilomenaQuery.Parse.Helpers, :to_number, []}) edit_distance = ignore(string("~")) diff --git a/lib/philomena_query/parse/numeric_parser.ex b/lib/philomena_query/parse/numeric_parser.ex new file mode 100644 index 000000000..95ff58be3 --- /dev/null +++ b/lib/philomena_query/parse/numeric_parser.ex @@ -0,0 +1,15 @@ +defmodule PhilomenaQuery.Parse.NumericParser do + @moduledoc false + + import NimbleParsec + + numeric_parser = + ascii_string([?0..?9], min: 1) + |> reduce({List, :to_string, []}) + |> reduce({PhilomenaQuery.Parse.Helpers, :to_int, []}) + |> unwrap_and_tag(:numeric) + |> eos() + |> label("a numeric value, like `3' or `10'") + + defparsec(:parse, numeric_parser) +end diff --git a/lib/philomena_query/parse/parser.ex b/lib/philomena_query/parse/parser.ex index 320316aae..ab1870055 100644 --- a/lib/philomena_query/parse/parser.ex +++ b/lib/philomena_query/parse/parser.ex @@ -14,6 +14,7 @@ defmodule PhilomenaQuery.Parse.Parser do - Dates (absolute and relative, time points and ranges) - Floats - Integers + - Numerics - IP Addresses - Literal text - Stemmed text @@ -33,6 +34,7 @@ defmodule PhilomenaQuery.Parse.Parser do DateParser, FloatParser, IntParser, + NumericParser, IpParser, Lexer, LiteralParser, @@ -90,6 +92,7 @@ defmodule PhilomenaQuery.Parse.Parser do date_fields: [String.t()], float_fields: [String.t()], int_fields: [String.t()], + numeric_fields: [String.t()], ip_fields: [String.t()], literal_fields: [String.t()], ngram_fields: [String.t()], @@ -108,6 +111,7 @@ defmodule PhilomenaQuery.Parse.Parser do date_fields: [], float_fields: [], int_fields: [], + numeric_fields: [], ip_fields: [], literal_fields: [], ngram_fields: [], @@ -131,7 +135,8 @@ defmodule PhilomenaQuery.Parse.Parser do Available options: - `bool_fields` - a list of field names parsed as booleans - `float_fields` - a list of field names parsed as floats - - `int_fields` - a list of field names parsed as integers + - `int_fields` - a list of field names parsed as signed integers + - `numeric_fields` - a list of field names parsed as numeric values (unsigned integers without fuzzing or range queries) - `ip_fields` - a list of field names parsed as IP CIDR masks - `literal_fields` - wildcardable fields which are searched as the exact value - `ngram_fields` - wildcardable fields which are searched as stemmed values @@ -164,6 +169,7 @@ defmodule PhilomenaQuery.Parse.Parser do Enum.map(parser.date_fields, fn f -> {f, DateParser} end) ++ Enum.map(parser.float_fields, fn f -> {f, FloatParser} end) ++ Enum.map(parser.int_fields, fn f -> {f, IntParser} end) ++ + Enum.map(parser.numeric_fields, fn f -> {f, NumericParser} end) ++ Enum.map(parser.ip_fields, fn f -> {f, IpParser} end) ++ Enum.map(parser.literal_fields, fn f -> {f, LiteralParser} end) ++ Enum.map(parser.ngram_fields, fn f -> {f, NgramParser} end) ++ @@ -437,6 +443,12 @@ defmodule PhilomenaQuery.Parse.Parser do defp field_type(_parser, [{IntParser, field_name}, range: _range, int_range: _value]), do: {:error, "multiple ranges specified for " <> field_name} + defp field_type(parser, [{NumericParser, field_name}, range: :eq, numeric: value]), + do: {:ok, {%{term: %{field(parser, field_name) => value}}, []}} + + defp field_type(_parser, [{NumericParser, field_name}, _range, _value]), + do: {:error, "range specified for " <> field_name} + defp field_type(parser, [{FloatParser, field_name}, range: :eq, float: value]), do: {:ok, {%{term: %{field(parser, field_name) => value}}, []}}