Cleanup indicies

config/aggregation.json

@@ -11,7 +11,7 @@
       "last_24h": {
         "filter": {
           "range": {
-            "posted_at": {
+            "created_at": {
               "gt": "now-24h"
             }
           }

lib/philomena/comments.ex

@@ -382,7 +382,8 @@ defmodule Philomena.Comments do
 
     [
       user: user_query,
-      image: image_query
+      image: image_query,
+      deleted_by: user_query
     ]
   end
 

lib/philomena/comments/comment.ex

@@ -57,6 +57,7 @@ defmodule Philomena.Comments.Comment do
   def unhide_changeset(comment) do
     change(comment)
     |> put_change(:hidden_from_users, false)
+    |> put_change(:deleted_by_id, nil)
     |> put_change(:deletion_reason, "")
   end
 

lib/philomena/comments/query.ex

@@ -1,89 +1,42 @@
 defmodule Philomena.Comments.Query do
   alias PhilomenaQuery.Parse.Parser
 
-  defp user_id_transform(_ctx, data) do
-    case Integer.parse(data) do
-      {int, _rest} ->
-        {
-          :ok,
-          %{
-            bool: %{
-              must: [
-                %{term: %{anonymous: false}},
-                %{term: %{user_id: int}}
-              ]
-            }
-          }
-        }
-
-      _err ->
-        {:error, "Unknown `user_id' value."}
-    end
-  end
-
-  defp author_transform(_ctx, data) do
-    {
-      :ok,
-      %{
-        bool: %{
-          must: [
-            %{term: %{anonymous: false}},
-            %{
-              bool: %{
-                should: [
-                  %{term: %{author: data}},
-                  %{wildcard: %{author: data}}
-                ]
-              }
-            }
-          ]
-        }
-      }
-    }
-  end
-
   defp user_my_transform(%{user: %{id: id}}, "comments"),
-    do: {:ok, %{term: %{user_id: id}}}
+    do: {:ok, %{term: %{true_author_id: id}}}
 
   defp user_my_transform(_ctx, _value),
     do: {:error, "Unknown `my' value."}
 
   defp anonymous_fields do
     [
       int_fields: ~W(id),
-      numeric_fields: ~W(image_id),
-      date_fields: ~W(created_at),
+      numeric_fields: ~W(author_id image_id),
+      date_fields: ~W(created_at updated_at),
+      literal_fields: ~W(author),
       ngram_fields: ~W(body),
-      custom_fields: ~W(author user_id),
-      default_field: {"body", :ngram},
-      transforms: %{
-        "user_id" => &user_id_transform/2,
-        "author" => &author_transform/2
-      },
-      aliases: %{"created_at" => "posted_at"}
+      default_field: {"body", :ngram}
     ]
   end
 
   defp user_fields do
     fields = anonymous_fields()
 
     Keyword.merge(fields,
-      custom_fields: fields[:custom_fields] ++ ~W(my),
-      transforms: Map.merge(fields[:transforms], %{"my" => &user_my_transform/2})
+      custom_fields: ~W(my),
+      transforms: %{"my" => &user_my_transform/2}
     )
   end
 
   defp moderator_fields do
     fields = user_fields()
 
     Keyword.merge(fields,
-      numeric_fields: fields[:numeric_fields] ++ ~W(user_id),
-      literal_fields: ~W(author fingerprint),
+      numeric_fields: fields[:numeric_fields] ++ ~W(true_author_id deleted_by_user_id),
+      literal_fields: fields[:literal_fields] ++ ~W(true_author fingerprint deleted_by_user),
+      ngram_fields: fields[:ngram_fields] ++ ~W(deletion_reason),
       ip_fields: ~W(ip),
-      bool_fields: ~W(anonymous deleted),
-      custom_fields: fields[:custom_fields] -- ~W(author user_id),
-      aliases: Map.merge(fields[:aliases], %{"deleted" => "hidden_from_users"}),
-      transforms: Map.drop(fields[:transforms], ~W(author user_id))
+      bool_fields: ~W(anonymous deleted destroyed_content),
+      aliases: %{"deleted" => "hidden_from_users"}
     )
   end
 

lib/philomena/comments/search_index.ex

@@ -19,19 +19,24 @@ defmodule Philomena.Comments.SearchIndex do
         dynamic: false,
         properties: %{
           id: %{type: "integer"},
-          posted_at: %{type: "date"},
+          created_at: %{type: "date"},
+          updated_at: %{type: "date"},
           ip: %{type: "ip"},
           fingerprint: %{type: "keyword"},
           image_id: %{type: "keyword"},
-          user_id: %{type: "keyword"},
+          author_id: %{type: "keyword"},
           author: %{type: "keyword"},
+          true_author_id: %{type: "keyword"},
+          true_author: %{type: "keyword"},
           image_tag_ids: %{type: "keyword"},
-          # boolean
-          anonymous: %{type: "keyword"},
-          # boolean
-          hidden_from_users: %{type: "keyword"},
+          anonymous: %{type: "boolean"},
+          hidden_from_users: %{type: "boolean"},
           body: %{type: "text", analyzer: "snowball"},
-          approved: %{type: "boolean"}
+          approved: %{type: "boolean"},
+          deleted_by_user: %{type: "keyword"},
+          deleted_by_user_id: %{type: "keyword"},
+          deletion_reason: %{type: "text", analyzer: "snowball"},
+          destroyed_content: %{type: "boolean"}
         }
       }
     }
@@ -41,24 +46,46 @@ defmodule Philomena.Comments.SearchIndex do
   def as_json(comment) do
     %{
       id: comment.id,
-      posted_at: comment.created_at,
-      ip: comment.ip |> to_string,
+      created_at: comment.created_at,
+      updated_at: comment.updated_at,
+      ip: to_string(comment.ip),
       fingerprint: comment.fingerprint,
       image_id: comment.image_id,
-      user_id: comment.user_id,
-      author: if(!!comment.user and !comment.anonymous, do: comment.user.name),
+      author_id: if(!comment.anonymous, do: comment.user_id),
+      author: if(!!comment.user and !comment.anonymous, do: String.downcase(comment.user.name)),
+      true_author_id: comment.user_id,
+      true_author: if(!!comment.user, do: String.downcase(comment.user.name)),
       image_tag_ids: comment.image.tags |> Enum.map(& &1.id),
       anonymous: comment.anonymous,
       hidden_from_users: comment.image.hidden_from_users || comment.hidden_from_users,
       body: comment.body,
-      approved: comment.image.approved && comment.approved
+      approved: comment.image.approved && comment.approved,
+      deleted_by_user: if(!!comment.deleted_by, do: String.downcase(comment.deleted_by.name)),
+      deleted_by_user_id: comment.deleted_by_id,
+      deletion_reason: comment.deletion_reason,
+      destroyed_content: comment.destroyed_content
     }
   end
 
   def user_name_update_by_query(old_name, new_name) do
+    old_name = String.downcase(old_name)
+    new_name = String.downcase(new_name)
+
     %{
-      query: %{term: %{author: old_name}},
-      replacements: [%{path: ["author"], old: old_name, new: new_name}],
+      query: %{
+        bool: %{
+          should: [
+            %{term: %{author: old_name}},
+            %{term: %{true_author: old_name}},
+            %{term: %{deleted_by_user: old_name}}
+          ]
+        }
+      },
+      replacements: [
+        %{path: ["author"], old: old_name, new: new_name},
+        %{path: ["true_author"], old: old_name, new: new_name},
+        %{path: ["deleted_by_user"], old: old_name, new: new_name}
+      ],
       set_replacements: []
     }
   end

lib/philomena/filters/search_index.ex

@@ -30,11 +30,8 @@ defmodule Philomena.Filters.SearchIndex do
           hidden_count: %{type: "integer"},
           spoilered_tag_ids: %{type: "keyword"},
           hidden_tag_ids: %{type: "keyword"},
-          spoilered_tags: %{type: "keyword"},
-          hidden_tags: %{type: "keyword"},
           spoilered_complex_str: %{type: "keyword"},
-          hidden_complex_str: %{type: "keyword"},
-          user_count: %{type: "integer"}
+          hidden_complex_str: %{type: "keyword"}
         }
       }
     }
@@ -58,8 +55,7 @@ defmodule Philomena.Filters.SearchIndex do
       spoilered_complex_str:
         if(!!filter.spoilered_complex_str, do: String.downcase(filter.spoilered_complex_str)),
       hidden_complex_str:
-        if(!!filter.hidden_complex_str, do: String.downcase(filter.hidden_complex_str)),
-      user_count: filter.user_count
+        if(!!filter.hidden_complex_str, do: String.downcase(filter.hidden_complex_str))
     }
   end
 

lib/philomena/galleries/query.ex

@@ -4,12 +4,13 @@ defmodule Philomena.Galleries.Query do
   defp fields do
     [
       int_fields: ~W(id image_count watcher_count),
-      numeric_fields: ~W(image_ids watcher_ids),
+      numeric_fields: ~W(user_id image_ids thumbnail_id),
       literal_fields: ~W(title user),
       date_fields: ~W(created_at updated_at),
-      ngram_fields: ~W(description),
+      ngram_fields: ~W(description spoiler_warning),
       default_field: {"title", :term},
       aliases: %{
+        "user_id" => "creator_id",
         "user" => "creator"
       }
     ]

lib/philomena/galleries/search_index.ex

@@ -18,19 +18,18 @@ defmodule Philomena.Galleries.SearchIndex do
       mappings: %{
         dynamic: false,
         properties: %{
-          # keyword
           id: %{type: "integer"},
           image_count: %{type: "integer"},
           watcher_count: %{type: "integer"},
           updated_at: %{type: "date"},
           created_at: %{type: "date"},
           title: %{type: "keyword"},
-          # missing creator_id
+          creator_id: %{type: "keyword"},
           creator: %{type: "keyword"},
           image_ids: %{type: "keyword"},
-          # ???
-          watcher_ids: %{type: "keyword"},
-          description: %{type: "text", analyzer: "snowball"}
+          description: %{type: "text", analyzer: "snowball"},
+          thumbnail_id: %{type: "keyword"},
+          spoiler_warning: %{type: "text", analyzer: "snowball"}
         }
       }
     }
@@ -42,13 +41,15 @@ defmodule Philomena.Galleries.SearchIndex do
       id: gallery.id,
       image_count: gallery.image_count,
       watcher_count: length(gallery.subscribers),
-      watcher_ids: Enum.map(gallery.subscribers, & &1.id),
       updated_at: gallery.updated_at,
       created_at: gallery.created_at,
       title: String.downcase(gallery.title),
+      creator_id: gallery.creator_id,
       creator: String.downcase(gallery.creator.name),
       image_ids: Enum.map(gallery.interactions, & &1.image_id),
-      description: gallery.description
+      description: gallery.description,
+      thumbnail_id: gallery.thumbnail_id,
+      spoiler_warning: gallery.spoiler_warning
     }
   end
 

lib/philomena/images/query.ex

@@ -120,7 +120,7 @@ defmodule Philomena.Images.Query do
       bool_fields: ~W(animated processed thumbnails_generated),
       ngram_fields: ~W(description),
       custom_fields: ~W(gallery_id filter_id),
-      default_field: {"namespaced_tags.name", :term},
+      default_field: {"tags", :term},
       transforms: %{
         "gallery_id" => &gallery_id_transform/2,
         "filter_id" => &filter_id_transform/2
@@ -130,7 +130,7 @@ defmodule Philomena.Images.Query do
         "faved_by_id" => "favourited_by_user_ids"
       },
       no_downcase_fields: ~W(file_name),
-      normalizations: %{"namespaced_tags.name" => &Tag.clean_tag_name/1}
+      normalizations: %{"tags" => &Tag.clean_tag_name/1}
     ]
   end
 
@@ -153,7 +153,7 @@ defmodule Philomena.Images.Query do
       literal_fields:
         fields[:literal_fields] ++
           ~W(fingerprint upvoted_by downvoted_by true_uploader hidden_by deleted_by_user),
-      ngram_fields: fields[:ngram_fields] ++ ~W(deletion_reason),
+      ngram_fields: fields[:ngram_fields] ++ ~W(deletion_reason scratchpad),
       ip_fields: ~W(ip),
       bool_fields: fields[:bool_fields] ++ ~W(anonymous deleted),
       aliases: