From bd652764211be63589f15491fed60c3c0b591dde Mon Sep 17 00:00:00 2001 From: mdashlw Date: Sun, 27 Apr 2025 19:19:53 +0800 Subject: [PATCH 01/11] Cleanup indicies --- config/aggregation.json | 2 +- lib/philomena/comments/query.ex | 9 ++++----- lib/philomena/comments/search_index.ex | 10 +++++----- lib/philomena/filters/search_index.ex | 2 -- lib/philomena/galleries/query.ex | 5 +++-- lib/philomena/galleries/search_index.ex | 15 ++++++++------- lib/philomena/images/query.ex | 2 +- lib/philomena/images/search_index.ex | 16 +++++----------- lib/philomena/posts/query.ex | 3 ++- lib/philomena/posts/search_index.ex | 10 +++------- lib/philomena/reports/search_index.ex | 2 +- lib/philomena/tags/search_index.ex | 2 -- .../controllers/activity_controller.ex | 4 ++-- .../api/json/search/comment_controller.ex | 2 +- .../api/json/search/post_controller.ex | 2 +- .../controllers/comment_controller.ex | 2 +- lib/philomena_web/controllers/post_controller.ex | 2 +- .../controllers/profile_controller.ex | 4 ++-- 18 files changed, 41 insertions(+), 53 deletions(-) diff --git a/config/aggregation.json b/config/aggregation.json index 3591fea55..c9366c106 100644 --- a/config/aggregation.json +++ b/config/aggregation.json @@ -11,7 +11,7 @@ "last_24h": { "filter": { "range": { - "posted_at": { + "created_at": { "gt": "now-24h" } } diff --git a/lib/philomena/comments/query.ex b/lib/philomena/comments/query.ex index 49ec6d688..f38e0e4bb 100644 --- a/lib/philomena/comments/query.ex +++ b/lib/philomena/comments/query.ex @@ -51,7 +51,7 @@ defmodule Philomena.Comments.Query do defp anonymous_fields do [ int_fields: ~W(id), - date_fields: ~W(created_at), + date_fields: ~W(created_at updated_at), literal_fields: ~W(image_id), ngram_fields: ~W(body), custom_fields: ~W(author user_id), @@ -59,8 +59,7 @@ defmodule Philomena.Comments.Query do transforms: %{ "user_id" => &user_id_transform/2, "author" => &author_transform/2 - }, - aliases: %{"created_at" => "posted_at"} + } ] end @@ -81,8 +80,8 @@ defmodule Philomena.Comments.Query do ip_fields: ~W(ip), bool_fields: ~W(anonymous deleted), custom_fields: fields[:custom_fields] -- ~W(author user_id), - aliases: Map.merge(fields[:aliases], %{"deleted" => "hidden_from_users"}), - transforms: Map.drop(fields[:transforms], ~W(author user_id)) + transforms: Map.drop(fields[:transforms], ~W(author user_id)), + aliases: %{"deleted" => "hidden_from_users"} ) end diff --git a/lib/philomena/comments/search_index.ex b/lib/philomena/comments/search_index.ex index b23589239..ace9db188 100644 --- a/lib/philomena/comments/search_index.ex +++ b/lib/philomena/comments/search_index.ex @@ -19,16 +19,15 @@ defmodule Philomena.Comments.SearchIndex do dynamic: false, properties: %{ id: %{type: "integer"}, - posted_at: %{type: "date"}, + created_at: %{type: "date"}, + updated_at: %{type: "date"}, ip: %{type: "ip"}, fingerprint: %{type: "keyword"}, image_id: %{type: "keyword"}, user_id: %{type: "keyword"}, author: %{type: "keyword"}, image_tag_ids: %{type: "keyword"}, - # boolean anonymous: %{type: "keyword"}, - # boolean hidden_from_users: %{type: "keyword"}, body: %{type: "text", analyzer: "snowball"}, approved: %{type: "boolean"} @@ -41,8 +40,9 @@ defmodule Philomena.Comments.SearchIndex do def as_json(comment) do %{ id: comment.id, - posted_at: comment.created_at, - ip: comment.ip |> to_string, + created_at: comment.created_at, + updated_at: comment.updated_at, + ip: to_string(comment.ip), fingerprint: comment.fingerprint, image_id: comment.image_id, user_id: comment.user_id, diff --git a/lib/philomena/filters/search_index.ex b/lib/philomena/filters/search_index.ex index 4d8256a00..c4b769174 100644 --- a/lib/philomena/filters/search_index.ex +++ b/lib/philomena/filters/search_index.ex @@ -30,8 +30,6 @@ defmodule Philomena.Filters.SearchIndex do hidden_count: %{type: "integer"}, spoilered_tag_ids: %{type: "keyword"}, hidden_tag_ids: %{type: "keyword"}, - spoilered_tags: %{type: "keyword"}, - hidden_tags: %{type: "keyword"}, spoilered_complex_str: %{type: "keyword"}, hidden_complex_str: %{type: "keyword"}, user_count: %{type: "integer"} diff --git a/lib/philomena/galleries/query.ex b/lib/philomena/galleries/query.ex index ddfa1e8bc..528d84035 100644 --- a/lib/philomena/galleries/query.ex +++ b/lib/philomena/galleries/query.ex @@ -4,11 +4,12 @@ defmodule Philomena.Galleries.Query do defp fields do [ int_fields: ~W(id image_count watcher_count), - literal_fields: ~W(title user image_ids watcher_ids), + literal_fields: ~W(title user_id user image_ids thumbnail_id), date_fields: ~W(created_at updated_at), - ngram_fields: ~W(description), + ngram_fields: ~W(description spoiler_warning), default_field: {"title", :term}, aliases: %{ + "user_id" => "creator_id", "user" => "creator" } ] diff --git a/lib/philomena/galleries/search_index.ex b/lib/philomena/galleries/search_index.ex index b3712cf1f..dbe640d42 100644 --- a/lib/philomena/galleries/search_index.ex +++ b/lib/philomena/galleries/search_index.ex @@ -18,19 +18,18 @@ defmodule Philomena.Galleries.SearchIndex do mappings: %{ dynamic: false, properties: %{ - # keyword id: %{type: "integer"}, image_count: %{type: "integer"}, watcher_count: %{type: "integer"}, updated_at: %{type: "date"}, created_at: %{type: "date"}, title: %{type: "keyword"}, - # missing creator_id + creator_id: %{type: "keyword"}, creator: %{type: "keyword"}, image_ids: %{type: "keyword"}, - # ??? - watcher_ids: %{type: "keyword"}, - description: %{type: "text", analyzer: "snowball"} + description: %{type: "text", analyzer: "snowball"}, + thumbnail_id: %{type: "keyword"}, + spoiler_warning: %{type: "text", analyzer: "snowball"} } } } @@ -42,13 +41,15 @@ defmodule Philomena.Galleries.SearchIndex do id: gallery.id, image_count: gallery.image_count, watcher_count: length(gallery.subscribers), - watcher_ids: Enum.map(gallery.subscribers, & &1.id), updated_at: gallery.updated_at, created_at: gallery.created_at, title: String.downcase(gallery.title), + creator_id: gallery.creator_id, creator: String.downcase(gallery.creator.name), image_ids: Enum.map(gallery.interactions, & &1.image_id), - description: gallery.description + description: gallery.description, + thumbnail_id: gallery.thumbnail_id, + spoiler_warning: gallery.spoiler_warning } end diff --git a/lib/philomena/images/query.ex b/lib/philomena/images/query.ex index 901147ac1..3c2d8c64c 100644 --- a/lib/philomena/images/query.ex +++ b/lib/philomena/images/query.ex @@ -124,7 +124,7 @@ defmodule Philomena.Images.Query do literal_fields: fields[:literal_fields] ++ ~W(fingerprint upvoted_by downvoted_by true_uploader hidden_by deleted_by_user), - ngram_fields: fields[:ngram_fields] ++ ~W(deletion_reason), + ngram_fields: fields[:ngram_fields] ++ ~W(deletion_reason scratchpad), ip_fields: ~W(ip), bool_fields: fields[:bool_fields] ++ ~W(anonymous deleted), aliases: diff --git a/lib/philomena/images/search_index.ex b/lib/philomena/images/search_index.ex index 35241ccde..990a975e8 100644 --- a/lib/philomena/images/search_index.ex +++ b/lib/philomena/images/search_index.ex @@ -22,7 +22,6 @@ defmodule Philomena.Images.SearchIndex do anonymous: %{type: "boolean"}, aspect_ratio: %{type: "float"}, comment_count: %{type: "integer"}, - commenters: %{type: "keyword"}, created_at: %{type: "date"}, deleted_by_user: %{type: "keyword"}, deleted_by_user_id: %{type: "keyword"}, @@ -39,8 +38,6 @@ defmodule Philomena.Images.SearchIndex do file_name: %{type: "keyword"}, fingerprint: %{type: "keyword"}, first_seen_at: %{type: "date"}, - fps: %{type: "float"}, - frames: %{type: "integer"}, height: %{type: "integer"}, hidden_by_user_ids: %{type: "keyword"}, hidden_by_users: %{type: "keyword"}, @@ -60,7 +57,6 @@ defmodule Philomena.Images.SearchIndex do source_count: %{type: "integer"}, tag_count: %{type: "integer"}, tag_ids: %{type: "keyword"}, - tags: %{type: "text", analyzer: "keyword"}, thumbnails_generated: %{type: "boolean"}, true_uploader: %{type: "keyword"}, true_uploader_id: %{type: "keyword"}, @@ -80,8 +76,6 @@ defmodule Philomena.Images.SearchIndex do position: %{type: "integer"} } }, - gallery_id: %{type: "keyword"}, - gallery_position: %{type: "object", enabled: false}, namespaced_tags: %{ properties: %{ name: %{type: "keyword"}, @@ -99,7 +93,8 @@ defmodule Philomena.Images.SearchIndex do body_type_tag_count: %{type: "integer"}, content_fanmade_tag_count: %{type: "integer"}, content_official_tag_count: %{type: "integer"}, - spoiler_tag_count: %{type: "integer"} + spoiler_tag_count: %{type: "integer"}, + scratchpad: %{type: "text", analyzer: "snowball"} } } } @@ -127,7 +122,7 @@ defmodule Philomena.Images.SearchIndex do created_at: image.created_at, updated_at: image.updated_at, first_seen_at: image.first_seen_at, - ip: image.ip |> to_string, + ip: to_string(image.ip), tag_ids: image.tags |> Enum.map(& &1.id), mime_type: image.image_mime_type, uploader: if(!!image.user and !image.anonymous, do: String.downcase(image.user.name)), @@ -158,8 +153,6 @@ defmodule Philomena.Images.SearchIndex do namespaced_tags: %{ name: image.tags |> Enum.flat_map(&([&1] ++ &1.aliases)) |> Enum.map(& &1.name) }, - gallery_id: Enum.map(image.gallery_interactions, & &1.gallery_id), - gallery_position: Map.new(image.gallery_interactions, &{&1.gallery_id, &1.position}), favourited_by_users: image.favers |> Enum.map(&String.downcase(&1.name)), hidden_by_users: image.hiders |> Enum.map(&String.downcase(&1.name)), upvoters: image.upvoters |> Enum.map(&String.downcase(&1.name)), @@ -175,7 +168,8 @@ defmodule Philomena.Images.SearchIndex do body_type_tag_count: Enum.count(image.tags, &(&1.category == "body-type")), content_fanmade_tag_count: Enum.count(image.tags, &(&1.category == "content-fanmade")), content_official_tag_count: Enum.count(image.tags, &(&1.category == "content-official")), - spoiler_tag_count: Enum.count(image.tags, &(&1.category == "spoiler")) + spoiler_tag_count: Enum.count(image.tags, &(&1.category == "spoiler")), + scratchpad: image.scratchpad } end diff --git a/lib/philomena/posts/query.ex b/lib/philomena/posts/query.ex index 58d94d6ca..77e75ad92 100644 --- a/lib/philomena/posts/query.ex +++ b/lib/philomena/posts/query.ex @@ -80,7 +80,8 @@ defmodule Philomena.Posts.Query do ip_fields: ~W(ip), bool_fields: ~W(anonymous deleted), custom_fields: fields[:custom_fields] -- ~W(author user_id), - transforms: Map.drop(fields[:transforms], ["user_id", "author"]) + transforms: Map.drop(fields[:transforms], ["user_id", "author"]), + aliases: %{"deleted" => "hidden_from_users"} ) end diff --git a/lib/philomena/posts/search_index.ex b/lib/philomena/posts/search_index.ex index b5522fa18..a9fefc8ae 100644 --- a/lib/philomena/posts/search_index.ex +++ b/lib/philomena/posts/search_index.ex @@ -21,8 +21,6 @@ defmodule Philomena.Posts.SearchIndex do id: %{type: "integer"}, body: %{type: "text", analyzer: "snowball"}, ip: %{type: "ip"}, - user_agent: %{type: "keyword"}, - referrer: %{type: "keyword"}, fingerprint: %{type: "keyword"}, subject: %{type: "text", analyzer: "snowball"}, author: %{type: "keyword"}, @@ -34,7 +32,7 @@ defmodule Philomena.Posts.SearchIndex do anonymous: %{type: "boolean"}, updated_at: %{type: "date"}, created_at: %{type: "date"}, - deleted: %{type: "boolean"}, + hidden_from_users: %{type: "boolean"}, access_level: %{type: "keyword"}, destroyed_content: %{type: "boolean"}, approved: %{type: "boolean"} @@ -51,9 +49,7 @@ defmodule Philomena.Posts.SearchIndex do body: post.body, author: if(!!post.user and !post.anonymous, do: String.downcase(post.user.name)), subject: post.topic.title, - ip: post.ip |> to_string(), - user_agent: "", - referrer: "", + ip: to_string(post.ip), fingerprint: post.fingerprint, topic_position: post.topic_position, forum: post.topic.forum.short_name, @@ -62,7 +58,7 @@ defmodule Philomena.Posts.SearchIndex do anonymous: post.anonymous, created_at: post.created_at, updated_at: post.updated_at, - deleted: post.hidden_from_users, + hidden_from_users: post.hidden_from_users, access_level: post.topic.forum.access_level, destroyed_content: post.destroyed_content, approved: post.approved diff --git a/lib/philomena/reports/search_index.ex b/lib/philomena/reports/search_index.ex index 61b988de9..61aa02f1b 100644 --- a/lib/philomena/reports/search_index.ex +++ b/lib/philomena/reports/search_index.ex @@ -44,7 +44,7 @@ defmodule Philomena.Reports.SearchIndex do id: report.id, image_id: image_id(report), created_at: report.created_at, - ip: report.ip |> to_string(), + ip: to_string(report.ip), state: report.state, user: if(report.user, do: String.downcase(report.user.name)), user_id: report.user_id, diff --git a/lib/philomena/tags/search_index.ex b/lib/philomena/tags/search_index.ex index 4589c0656..3cf40a8db 100644 --- a/lib/philomena/tags/search_index.ex +++ b/lib/philomena/tags/search_index.ex @@ -35,7 +35,6 @@ defmodule Philomena.Tags.SearchIndex do aliased_tag: %{type: "keyword"}, aliases: %{type: "keyword"}, implied_tags: %{type: "keyword"}, - implied_tag_ids: %{type: "keyword"}, implied_by_tags: %{type: "keyword"}, category: %{type: "keyword"}, aliased: %{type: "boolean"}, @@ -64,7 +63,6 @@ defmodule Philomena.Tags.SearchIndex do namespace: tag.namespace, analyzed_name: tag.name, implied_tags: tag.implied_tags |> Enum.map(& &1.name), - implied_tag_ids: tag.implied_tags |> Enum.map(& &1.id), implied_by_tags: tag.implied_by_tags |> Enum.map(& &1.name), aliased_tag: if(!!tag.aliased_tag, do: tag.aliased_tag.name), aliases: tag.aliases |> Enum.map(& &1.name), diff --git a/lib/philomena_web/controllers/activity_controller.ex b/lib/philomena_web/controllers/activity_controller.ex index 3e85233c8..d9bc5b5c3 100644 --- a/lib/philomena_web/controllers/activity_controller.ex +++ b/lib/philomena_web/controllers/activity_controller.ex @@ -42,7 +42,7 @@ defmodule PhilomenaWeb.ActivityController do query: %{ bool: %{ must: %{ - range: %{posted_at: %{gt: "now-1w"}} + range: %{created_at: %{gt: "now-1w"}} }, must_not: [ %{terms: %{image_tag_ids: conn.assigns.current_filter.hidden_tag_ids}}, @@ -50,7 +50,7 @@ defmodule PhilomenaWeb.ActivityController do ] } }, - sort: %{posted_at: :desc} + sort: %{created_at: :desc} }, %{page_number: 1, page_size: 6} ) diff --git a/lib/philomena_web/controllers/api/json/search/comment_controller.ex b/lib/philomena_web/controllers/api/json/search/comment_controller.ex index 6942a4ffa..e5ba08f49 100644 --- a/lib/philomena_web/controllers/api/json/search/comment_controller.ex +++ b/lib/philomena_web/controllers/api/json/search/comment_controller.ex @@ -27,7 +27,7 @@ defmodule PhilomenaWeb.Api.Json.Search.CommentController do } } }, - sort: %{posted_at: :desc} + sort: %{created_at: :desc} }, conn.assigns.pagination ) diff --git a/lib/philomena_web/controllers/api/json/search/post_controller.ex b/lib/philomena_web/controllers/api/json/search/post_controller.ex index c305de129..31f28ed7a 100644 --- a/lib/philomena_web/controllers/api/json/search/post_controller.ex +++ b/lib/philomena_web/controllers/api/json/search/post_controller.ex @@ -19,7 +19,7 @@ defmodule PhilomenaWeb.Api.Json.Search.PostController do bool: %{ must: [ query, - %{term: %{deleted: false}}, + %{term: %{hidden_from_users: false}}, %{term: %{access_level: "normal"}} ] } diff --git a/lib/philomena_web/controllers/comment_controller.ex b/lib/philomena_web/controllers/comment_controller.ex index d64f3e16b..f64d79dfe 100644 --- a/lib/philomena_web/controllers/comment_controller.ex +++ b/lib/philomena_web/controllers/comment_controller.ex @@ -31,7 +31,7 @@ defmodule PhilomenaWeb.CommentController do } } }, - sort: %{posted_at: :desc} + sort: %{created_at: :desc} }, conn.assigns.pagination ) diff --git a/lib/philomena_web/controllers/post_controller.ex b/lib/philomena_web/controllers/post_controller.ex index 6f00ff7c4..3d711e09a 100644 --- a/lib/philomena_web/controllers/post_controller.ex +++ b/lib/philomena_web/controllers/post_controller.ex @@ -58,7 +58,7 @@ defmodule PhilomenaWeb.PostController do defp filters(_user) do [ %{term: %{access_level: "normal"}}, - %{term: %{deleted: false}} + %{term: %{hidden_from_users: false}} ] end end diff --git a/lib/philomena_web/controllers/profile_controller.ex b/lib/philomena_web/controllers/profile_controller.ex index d3e375f49..6f3bd5ff3 100644 --- a/lib/philomena_web/controllers/profile_controller.ex +++ b/lib/philomena_web/controllers/profile_controller.ex @@ -94,7 +94,7 @@ defmodule PhilomenaWeb.ProfileController do ] } }, - sort: %{posted_at: :desc} + sort: %{created_at: :desc} }, %{page_size: 3} ) @@ -108,7 +108,7 @@ defmodule PhilomenaWeb.ProfileController do must: [ %{term: %{user_id: user.id}}, %{term: %{anonymous: false}}, - %{term: %{deleted: false}}, + %{term: %{hidden_from_users: false}}, %{term: %{access_level: "normal"}} ] } From 163747bbb4e29e4e06afb3712cc34dfea0b227b0 Mon Sep 17 00:00:00 2001 From: mdashlw Date: Sun, 27 Apr 2025 21:01:20 +0800 Subject: [PATCH 02/11] Index some fields as boolean --- lib/philomena/comments/search_index.ex | 4 ++-- lib/philomena/images/search_index.ex | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/philomena/comments/search_index.ex b/lib/philomena/comments/search_index.ex index ace9db188..62f0c29dd 100644 --- a/lib/philomena/comments/search_index.ex +++ b/lib/philomena/comments/search_index.ex @@ -27,8 +27,8 @@ defmodule Philomena.Comments.SearchIndex do user_id: %{type: "keyword"}, author: %{type: "keyword"}, image_tag_ids: %{type: "keyword"}, - anonymous: %{type: "keyword"}, - hidden_from_users: %{type: "keyword"}, + anonymous: %{type: "boolean"}, + hidden_from_users: %{type: "boolean"}, body: %{type: "text", analyzer: "snowball"}, approved: %{type: "boolean"} } diff --git a/lib/philomena/images/search_index.ex b/lib/philomena/images/search_index.ex index 990a975e8..59b012569 100644 --- a/lib/philomena/images/search_index.ex +++ b/lib/philomena/images/search_index.ex @@ -41,7 +41,7 @@ defmodule Philomena.Images.SearchIndex do height: %{type: "integer"}, hidden_by_user_ids: %{type: "keyword"}, hidden_by_users: %{type: "keyword"}, - hidden_from_users: %{type: "keyword"}, + hidden_from_users: %{type: "boolean"}, id: %{type: "integer"}, ip: %{type: "ip"}, mime_type: %{type: "keyword"}, From 64dbd8695e804e5db771fe02d7b9ebb27efd900e Mon Sep 17 00:00:00 2001 From: mdashlw Date: Sun, 27 Apr 2025 21:40:38 +0800 Subject: [PATCH 03/11] Index more fields on posts & comments --- lib/philomena/comments/query.ex | 5 ++++- lib/philomena/comments/search_index.ex | 29 +++++++++++++++++++++----- lib/philomena/posts/query.ex | 5 ++++- lib/philomena/posts/search_index.ex | 26 ++++++++++++++++++----- 4 files changed, 53 insertions(+), 12 deletions(-) diff --git a/lib/philomena/comments/query.ex b/lib/philomena/comments/query.ex index f38e0e4bb..ff378e48b 100644 --- a/lib/philomena/comments/query.ex +++ b/lib/philomena/comments/query.ex @@ -76,7 +76,10 @@ defmodule Philomena.Comments.Query do fields = user_fields() Keyword.merge(fields, - literal_fields: ~W(image_id user_id author fingerprint), + literal_fields: + fields[:literal_fields] ++ + ~W(user_id author fingerprint deleted_by_user_id deleted_by_user), + ngram_fields: fields[:ngram_fields] ++ ~W(deletion_reason), ip_fields: ~W(ip), bool_fields: ~W(anonymous deleted), custom_fields: fields[:custom_fields] -- ~W(author user_id), diff --git a/lib/philomena/comments/search_index.ex b/lib/philomena/comments/search_index.ex index 62f0c29dd..fe9c51d48 100644 --- a/lib/philomena/comments/search_index.ex +++ b/lib/philomena/comments/search_index.ex @@ -30,7 +30,10 @@ defmodule Philomena.Comments.SearchIndex do anonymous: %{type: "boolean"}, hidden_from_users: %{type: "boolean"}, body: %{type: "text", analyzer: "snowball"}, - approved: %{type: "boolean"} + approved: %{type: "boolean"}, + deleted_by_user: %{type: "keyword"}, + deleted_by_user_id: %{type: "keyword"}, + deletion_reason: %{type: "text", analyzer: "snowball"} } } } @@ -46,19 +49,35 @@ defmodule Philomena.Comments.SearchIndex do fingerprint: comment.fingerprint, image_id: comment.image_id, user_id: comment.user_id, - author: if(!!comment.user and !comment.anonymous, do: comment.user.name), + author: String.downcase(comment.user.name), image_tag_ids: comment.image.tags |> Enum.map(& &1.id), anonymous: comment.anonymous, hidden_from_users: comment.image.hidden_from_users || comment.hidden_from_users, body: comment.body, - approved: comment.image.approved && comment.approved + approved: comment.image.approved && comment.approved, + deleted_by_user: if(!!comment.deleted_by, do: String.downcase(comment.deleted_by.name)), + deleted_by_user_id: comment.deleted_by_id, + deletion_reason: comment.deletion_reason } end def user_name_update_by_query(old_name, new_name) do + old_name = String.downcase(old_name) + new_name = String.downcase(new_name) + %{ - query: %{term: %{author: old_name}}, - replacements: [%{path: ["author"], old: old_name, new: new_name}], + query: %{ + bool: %{ + should: [ + %{term: %{author: old_name}}, + %{term: %{deleted_by_user: old_name}} + ] + } + }, + replacements: [ + %{path: ["author"], old: old_name, new: new_name}, + %{path: ["deleted_by_user"], old: old_name, new: new_name} + ], set_replacements: [] } end diff --git a/lib/philomena/posts/query.ex b/lib/philomena/posts/query.ex index 77e75ad92..7bf20485e 100644 --- a/lib/philomena/posts/query.ex +++ b/lib/philomena/posts/query.ex @@ -76,7 +76,10 @@ defmodule Philomena.Posts.Query do fields = user_fields() Keyword.merge(fields, - literal_fields: fields[:literal_fields] ++ ~W(user_id author fingerprint), + literal_fields: + fields[:literal_fields] ++ + ~W(user_id author fingerprint deleted_by_user_id deleted_by_user), + ngram_fields: fields[:ngram_fields] ++ ~W(deletion_reason), ip_fields: ~W(ip), bool_fields: ~W(anonymous deleted), custom_fields: fields[:custom_fields] -- ~W(author user_id), diff --git a/lib/philomena/posts/search_index.ex b/lib/philomena/posts/search_index.ex index a9fefc8ae..fe2280824 100644 --- a/lib/philomena/posts/search_index.ex +++ b/lib/philomena/posts/search_index.ex @@ -35,7 +35,10 @@ defmodule Philomena.Posts.SearchIndex do hidden_from_users: %{type: "boolean"}, access_level: %{type: "keyword"}, destroyed_content: %{type: "boolean"}, - approved: %{type: "boolean"} + approved: %{type: "boolean"}, + deleted_by_user: %{type: "keyword"}, + deleted_by_user_id: %{type: "keyword"}, + deletion_reason: %{type: "text", analyzer: "snowball"} } } } @@ -47,7 +50,7 @@ defmodule Philomena.Posts.SearchIndex do id: post.id, topic_id: post.topic_id, body: post.body, - author: if(!!post.user and !post.anonymous, do: String.downcase(post.user.name)), + author: String.downcase(post.user.name), subject: post.topic.title, ip: to_string(post.ip), fingerprint: post.fingerprint, @@ -61,7 +64,10 @@ defmodule Philomena.Posts.SearchIndex do hidden_from_users: post.hidden_from_users, access_level: post.topic.forum.access_level, destroyed_content: post.destroyed_content, - approved: post.approved + approved: post.approved, + deleted_by_user: if(!!post.deleted_by, do: String.downcase(post.deleted_by.name)), + deleted_by_user_id: post.deleted_by_id, + deletion_reason: post.deletion_reason } end @@ -70,8 +76,18 @@ defmodule Philomena.Posts.SearchIndex do new_name = String.downcase(new_name) %{ - query: %{term: %{author: old_name}}, - replacements: [%{path: ["author"], old: old_name, new: new_name}], + query: %{ + bool: %{ + should: [ + %{term: %{author: old_name}}, + %{term: %{deleted_by_user: old_name}} + ] + } + }, + replacements: [ + %{path: ["author"], old: old_name, new: new_name}, + %{path: ["deleted_by_user"], old: old_name, new: new_name} + ], set_replacements: [] } end From 89525ccf127e5ef325d2da1fd6dd154c7e285bd2 Mon Sep 17 00:00:00 2001 From: mdashlw Date: Sun, 27 Apr 2025 21:51:06 +0800 Subject: [PATCH 04/11] Fix author indexing --- lib/philomena/comments/search_index.ex | 2 +- lib/philomena/posts/search_index.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/philomena/comments/search_index.ex b/lib/philomena/comments/search_index.ex index fe9c51d48..f29d86278 100644 --- a/lib/philomena/comments/search_index.ex +++ b/lib/philomena/comments/search_index.ex @@ -49,7 +49,7 @@ defmodule Philomena.Comments.SearchIndex do fingerprint: comment.fingerprint, image_id: comment.image_id, user_id: comment.user_id, - author: String.downcase(comment.user.name), + author: if(!!comment.user, do: String.downcase(comment.user.name)), image_tag_ids: comment.image.tags |> Enum.map(& &1.id), anonymous: comment.anonymous, hidden_from_users: comment.image.hidden_from_users || comment.hidden_from_users, diff --git a/lib/philomena/posts/search_index.ex b/lib/philomena/posts/search_index.ex index fe2280824..1e0b240c9 100644 --- a/lib/philomena/posts/search_index.ex +++ b/lib/philomena/posts/search_index.ex @@ -50,7 +50,7 @@ defmodule Philomena.Posts.SearchIndex do id: post.id, topic_id: post.topic_id, body: post.body, - author: String.downcase(post.user.name), + author: if(!!post.user, do: String.downcase(post.user.name)), subject: post.topic.title, ip: to_string(post.ip), fingerprint: post.fingerprint, From ae53e67a23945bb39c7d206878ded342f1c8c240 Mon Sep 17 00:00:00 2001 From: mdashlw Date: Mon, 28 Apr 2025 10:38:43 +0800 Subject: [PATCH 05/11] Simplify namespaced_tags --- lib/philomena/images/query.ex | 4 ++-- lib/philomena/images/search_index.ex | 12 ++---------- lib/philomena_query/parse/parser.ex | 6 +++--- lib/philomena_web/controllers/tag_controller.ex | 4 ++-- lib/philomena_web/image_loader.ex | 4 ++-- lib/philomena_web/views/image_view.ex | 3 +-- 6 files changed, 12 insertions(+), 21 deletions(-) diff --git a/lib/philomena/images/query.ex b/lib/philomena/images/query.ex index 3c2d8c64c..97b871e74 100644 --- a/lib/philomena/images/query.ex +++ b/lib/philomena/images/query.ex @@ -94,14 +94,14 @@ defmodule Philomena.Images.Query do bool_fields: ~W(animated processed thumbnails_generated), ngram_fields: ~W(description), custom_fields: ~W(gallery_id), - default_field: {"namespaced_tags.name", :term}, + default_field: {"tags", :term}, transforms: %{"gallery_id" => &gallery_id_transform/2}, aliases: %{ "faved_by" => "favourited_by_users", "faved_by_id" => "favourited_by_user_ids" }, no_downcase_fields: ~W(file_name), - normalizations: %{"namespaced_tags.name" => &Tag.clean_tag_name/1} + normalizations: %{"tags" => &Tag.clean_tag_name/1} ] end diff --git a/lib/philomena/images/search_index.ex b/lib/philomena/images/search_index.ex index 59b012569..466a1973f 100644 --- a/lib/philomena/images/search_index.ex +++ b/lib/philomena/images/search_index.ex @@ -76,13 +76,7 @@ defmodule Philomena.Images.SearchIndex do position: %{type: "integer"} } }, - namespaced_tags: %{ - properties: %{ - name: %{type: "keyword"}, - name_in_namespace: %{type: "keyword"}, - namespace: %{type: "keyword"} - } - }, + tags: %{type: "keyword"}, approved: %{type: "boolean"}, error_tag_count: %{type: "integer"}, rating_tag_count: %{type: "integer"}, @@ -150,9 +144,7 @@ defmodule Philomena.Images.SearchIndex do duplicate_id: image.duplicate_id, galleries: image.gallery_interactions |> Enum.map(&%{id: &1.gallery_id, position: &1.position}), - namespaced_tags: %{ - name: image.tags |> Enum.flat_map(&([&1] ++ &1.aliases)) |> Enum.map(& &1.name) - }, + tags: image.tags |> Enum.flat_map(&([&1] ++ &1.aliases)) |> Enum.map(& &1.name), favourited_by_users: image.favers |> Enum.map(&String.downcase(&1.name)), hidden_by_users: image.hiders |> Enum.map(&String.downcase(&1.name)), upvoters: image.upvoters |> Enum.map(&String.downcase(&1.name)), diff --git a/lib/philomena_query/parse/parser.ex b/lib/philomena_query/parse/parser.ex index 320316aae..112f1e7c4 100644 --- a/lib/philomena_query/parse/parser.ex +++ b/lib/philomena_query/parse/parser.ex @@ -182,15 +182,15 @@ defmodule PhilomenaQuery.Parse.Parser do ## Example iex> Parser.parse(parser, "safe") - {:ok, %{term: %{"namespaced_tags.name" => "safe"}}} + {:ok, %{term: %{"tags" => "safe"}}} iex> Parser.parse(nil, "safe OR solo") {:ok, %{ bool: %{ should: [ - %{term: %{"namespaced_tags.name" => "safe"}}, - %{term: %{"namespaced_tags.name" => "solo"}} + %{term: %{"tags" => "safe"}}, + %{term: %{"tags" => "solo"}} ] } }} diff --git a/lib/philomena_web/controllers/tag_controller.ex b/lib/philomena_web/controllers/tag_controller.ex index 4900ddd39..592897678 100644 --- a/lib/philomena_web/controllers/tag_controller.ex +++ b/lib/philomena_web/controllers/tag_controller.ex @@ -55,7 +55,7 @@ defmodule PhilomenaWeb.TagController do user = conn.assigns.current_user tag = conn.assigns.tag - {images, _tags} = ImageLoader.query(conn, %{term: %{"namespaced_tags.name" => tag.name}}) + {images, _tags} = ImageLoader.query(conn, %{term: %{"tags" => tag.name}}) images = Search.search_records(images, preload(Image, [:sources, tags: :aliases])) @@ -122,7 +122,7 @@ defmodule PhilomenaWeb.TagController do |> String.downcase() case Images.Query.compile(name) do - {:ok, %{term: %{"namespaced_tags.name" => ^name}}} -> + {:ok, %{term: %{"tags" => ^name}}} -> name _error -> diff --git a/lib/philomena_web/image_loader.ex b/lib/philomena_web/image_loader.ex index 81271e05b..7b73f5c2f 100644 --- a/lib/philomena_web/image_loader.ex +++ b/lib/philomena_web/image_loader.ex @@ -100,7 +100,7 @@ defmodule PhilomenaWeb.ImageLoader do do: [%{term: %{approved: false}} | filters] # TODO: the search parser should try to optimize queries - defp search_tag_name(%{term: %{"namespaced_tags.name" => tag_name}}), do: [tag_name] + defp search_tag_name(%{term: %{"tags" => tag_name}}), do: [tag_name] defp search_tag_name(_other_query), do: [] defp search_tag_names(%{bool: %{must: musts}}), do: Enum.flat_map(musts, &search_tag_name(&1)) @@ -108,7 +108,7 @@ defmodule PhilomenaWeb.ImageLoader do defp search_tag_names(%{bool: %{should: shoulds}}), do: Enum.flat_map(shoulds, &search_tag_name(&1)) - defp search_tag_names(%{term: %{"namespaced_tags.name" => tag_name}}), do: [tag_name] + defp search_tag_names(%{term: %{"tags" => tag_name}}), do: [tag_name] defp search_tag_names(_other_query), do: [] defp load_tags([]), do: [] diff --git a/lib/philomena_web/views/image_view.ex b/lib/philomena_web/views/image_view.ex index a10cad01f..4a26aac46 100644 --- a/lib/philomena_web/views/image_view.ex +++ b/lib/philomena_web/views/image_view.ex @@ -249,8 +249,7 @@ defmodule PhilomenaWeb.ImageView do def image_filter_data(image) do %{ id: image.id, - "namespaced_tags.name": - image.tags |> Enum.flat_map(&([&1] ++ &1.aliases)) |> Enum.map_join(", ", & &1.name), + tags: image.tags |> Enum.flat_map(&([&1] ++ &1.aliases)) |> Enum.map_join(", ", & &1.name), tag_count: length(image.tags), score: image.score, faves: image.faves_count, From 29faea10233c82270d57ad4bbac06b50127b342a Mon Sep 17 00:00:00 2001 From: mdashlw Date: Mon, 28 Apr 2025 13:38:33 +0800 Subject: [PATCH 06/11] Add destroyed_content --- lib/philomena/comments/query.ex | 2 +- lib/philomena/comments/search_index.ex | 6 ++++-- lib/philomena/posts/query.ex | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/philomena/comments/query.ex b/lib/philomena/comments/query.ex index ff378e48b..d9c6918e2 100644 --- a/lib/philomena/comments/query.ex +++ b/lib/philomena/comments/query.ex @@ -81,7 +81,7 @@ defmodule Philomena.Comments.Query do ~W(user_id author fingerprint deleted_by_user_id deleted_by_user), ngram_fields: fields[:ngram_fields] ++ ~W(deletion_reason), ip_fields: ~W(ip), - bool_fields: ~W(anonymous deleted), + bool_fields: ~W(anonymous deleted destroyed_content), custom_fields: fields[:custom_fields] -- ~W(author user_id), transforms: Map.drop(fields[:transforms], ~W(author user_id)), aliases: %{"deleted" => "hidden_from_users"} diff --git a/lib/philomena/comments/search_index.ex b/lib/philomena/comments/search_index.ex index f29d86278..7a9c7c8a5 100644 --- a/lib/philomena/comments/search_index.ex +++ b/lib/philomena/comments/search_index.ex @@ -33,7 +33,8 @@ defmodule Philomena.Comments.SearchIndex do approved: %{type: "boolean"}, deleted_by_user: %{type: "keyword"}, deleted_by_user_id: %{type: "keyword"}, - deletion_reason: %{type: "text", analyzer: "snowball"} + deletion_reason: %{type: "text", analyzer: "snowball"}, + destroyed_content: %{type: "boolean"} } } } @@ -57,7 +58,8 @@ defmodule Philomena.Comments.SearchIndex do approved: comment.image.approved && comment.approved, deleted_by_user: if(!!comment.deleted_by, do: String.downcase(comment.deleted_by.name)), deleted_by_user_id: comment.deleted_by_id, - deletion_reason: comment.deletion_reason + deletion_reason: comment.deletion_reason, + destroyed_content: comment.destroyed_content } end diff --git a/lib/philomena/posts/query.ex b/lib/philomena/posts/query.ex index 7bf20485e..e3171bac0 100644 --- a/lib/philomena/posts/query.ex +++ b/lib/philomena/posts/query.ex @@ -81,7 +81,7 @@ defmodule Philomena.Posts.Query do ~W(user_id author fingerprint deleted_by_user_id deleted_by_user), ngram_fields: fields[:ngram_fields] ++ ~W(deletion_reason), ip_fields: ~W(ip), - bool_fields: ~W(anonymous deleted), + bool_fields: ~W(anonymous deleted destroyed_content), custom_fields: fields[:custom_fields] -- ~W(author user_id), transforms: Map.drop(fields[:transforms], ["user_id", "author"]), aliases: %{"deleted" => "hidden_from_users"} From 965790ff6e5545b0db002b9ddb6740a8f17b6e66 Mon Sep 17 00:00:00 2001 From: mdashlw Date: Mon, 28 Apr 2025 15:38:46 +0800 Subject: [PATCH 07/11] Preload deleted_by for comment/post indexing --- lib/philomena/comments.ex | 3 ++- lib/philomena/posts.ex | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/philomena/comments.ex b/lib/philomena/comments.ex index b4b29fc8c..b6bea5aae 100644 --- a/lib/philomena/comments.ex +++ b/lib/philomena/comments.ex @@ -366,7 +366,8 @@ defmodule Philomena.Comments do [ user: user_query, - image: image_query + image: image_query, + deleted_by: user_query ] end diff --git a/lib/philomena/posts.ex b/lib/philomena/posts.ex index 70b696317..9d37ae062 100644 --- a/lib/philomena/posts.ex +++ b/lib/philomena/posts.ex @@ -380,7 +380,8 @@ defmodule Philomena.Posts do [ user: user_query, - topic: topic_query + topic: topic_query, + deleted_by: user_query ] end From 7915e426930a9f44a64eb8e97f0696372bcfc715 Mon Sep 17 00:00:00 2001 From: mdashlw Date: Mon, 28 Apr 2025 15:39:08 +0800 Subject: [PATCH 08/11] Change deleted_by_id to nil on unhide for posts/comments --- lib/philomena/comments/comment.ex | 1 + lib/philomena/posts/post.ex | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/philomena/comments/comment.ex b/lib/philomena/comments/comment.ex index e54a415b2..b25b6c737 100644 --- a/lib/philomena/comments/comment.ex +++ b/lib/philomena/comments/comment.ex @@ -57,6 +57,7 @@ defmodule Philomena.Comments.Comment do def unhide_changeset(comment) do change(comment) |> put_change(:hidden_from_users, false) + |> put_change(:deleted_by_id, nil) |> put_change(:deletion_reason, "") end diff --git a/lib/philomena/posts/post.ex b/lib/philomena/posts/post.ex index 11fc87bf5..cb4d021f3 100644 --- a/lib/philomena/posts/post.ex +++ b/lib/philomena/posts/post.ex @@ -72,6 +72,7 @@ defmodule Philomena.Posts.Post do def unhide_changeset(post) do change(post) |> put_change(:hidden_from_users, false) + |> put_change(:deleted_by_id, nil) |> put_change(:deletion_reason, "") end From 65bda96a4c9c1c15a5efe5d06179682ddfe13254 Mon Sep 17 00:00:00 2001 From: mdashlw Date: Tue, 29 Apr 2025 10:54:00 +0800 Subject: [PATCH 09/11] Add "system" field to reports query parser --- lib/philomena/reports/query.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/philomena/reports/query.ex b/lib/philomena/reports/query.ex index ebff3bf9d..86451b2f6 100644 --- a/lib/philomena/reports/query.ex +++ b/lib/philomena/reports/query.ex @@ -8,7 +8,7 @@ defmodule Philomena.Reports.Query do literal_fields: ~W(state user user_id admin admin_id reportable_type reportable_id fingerprint), ip_fields: ~W(ip), - bool_fields: ~W(open), + bool_fields: ~W(open system), ngram_fields: ~W(reason), default_field: {"reason", :ngram}, no_downcase_fields: ~W(reportable_type) From 2e5ed294e9bc2b4f3573f7ecc9ec102359c2e441 Mon Sep 17 00:00:00 2001 From: mdashlw Date: Wed, 30 Apr 2025 09:22:30 +0800 Subject: [PATCH 10/11] Remove user_count from filters --- lib/philomena/filters/search_index.ex | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/philomena/filters/search_index.ex b/lib/philomena/filters/search_index.ex index c4b769174..c6f451288 100644 --- a/lib/philomena/filters/search_index.ex +++ b/lib/philomena/filters/search_index.ex @@ -31,8 +31,7 @@ defmodule Philomena.Filters.SearchIndex do spoilered_tag_ids: %{type: "keyword"}, hidden_tag_ids: %{type: "keyword"}, spoilered_complex_str: %{type: "keyword"}, - hidden_complex_str: %{type: "keyword"}, - user_count: %{type: "integer"} + hidden_complex_str: %{type: "keyword"} } } } @@ -56,8 +55,7 @@ defmodule Philomena.Filters.SearchIndex do spoilered_complex_str: if(!!filter.spoilered_complex_str, do: String.downcase(filter.spoilered_complex_str)), hidden_complex_str: - if(!!filter.hidden_complex_str, do: String.downcase(filter.hidden_complex_str)), - user_count: filter.user_count + if(!!filter.hidden_complex_str, do: String.downcase(filter.hidden_complex_str)) } end From c7a90b0a9577cb2974273a19e08a4688db4f9271 Mon Sep 17 00:00:00 2001 From: mdashlw Date: Mon, 5 May 2025 09:43:11 +0800 Subject: [PATCH 11/11] Fix missing downcase --- lib/philomena/comments/query.ex | 2 ++ lib/philomena/posts/query.ex | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/philomena/comments/query.ex b/lib/philomena/comments/query.ex index ebd4db3bf..a882fd269 100644 --- a/lib/philomena/comments/query.ex +++ b/lib/philomena/comments/query.ex @@ -22,6 +22,8 @@ defmodule Philomena.Comments.Query do end defp author_transform(_ctx, data) do + data = String.downcase(data) + { :ok, %{ diff --git a/lib/philomena/posts/query.ex b/lib/philomena/posts/query.ex index 5a2fea658..9729e8047 100644 --- a/lib/philomena/posts/query.ex +++ b/lib/philomena/posts/query.ex @@ -22,6 +22,8 @@ defmodule Philomena.Posts.Query do end defp author_transform(_ctx, data) do + data = String.downcase(data) + { :ok, %{ @@ -31,8 +33,8 @@ defmodule Philomena.Posts.Query do %{ bool: %{ should: [ - %{term: %{author: String.downcase(data)}}, - %{wildcard: %{author: String.downcase(data)}} + %{term: %{author: data}}, + %{wildcard: %{author: data}} ] } }