Cleanup indicies

config/aggregation.json

@@ -11,7 +11,7 @@
       "last_24h": {
         "filter": {
           "range": {
-            "posted_at": {
+            "created_at": {
               "gt": "now-24h"
             }
           }

lib/philomena/comments.ex

@@ -366,7 +366,8 @@ defmodule Philomena.Comments do
 
     [
       user: user_query,
-      image: image_query
+      image: image_query,
+      deleted_by: user_query
     ]
   end
 

lib/philomena/comments/comment.ex

@@ -57,6 +57,7 @@ defmodule Philomena.Comments.Comment do
   def unhide_changeset(comment) do
     change(comment)
     |> put_change(:hidden_from_users, false)
+    |> put_change(:deleted_by_id, nil)
     |> put_change(:deletion_reason, "")
   end
 

lib/philomena/comments/query.ex

@@ -22,6 +22,8 @@ defmodule Philomena.Comments.Query do
   end
 
   defp author_transform(_ctx, data) do
+    data = String.downcase(data)
+
     {
       :ok,
       %{
@@ -52,15 +54,14 @@ defmodule Philomena.Comments.Query do
     [
       int_fields: ~W(id),
       numeric_fields: ~W(image_id),
-      date_fields: ~W(created_at),
+      date_fields: ~W(created_at updated_at),
       ngram_fields: ~W(body),
       custom_fields: ~W(author user_id),
       default_field: {"body", :ngram},
       transforms: %{
         "user_id" => &user_id_transform/2,
         "author" => &author_transform/2
-      },
-      aliases: %{"created_at" => "posted_at"}
+      }
     ]
   end
 
@@ -77,13 +78,14 @@ defmodule Philomena.Comments.Query do
     fields = user_fields()
 
     Keyword.merge(fields,
-      numeric_fields: fields[:numeric_fields] ++ ~W(user_id),
-      literal_fields: ~W(author fingerprint),
+      numeric_fields: fields[:numeric_fields] ++ ~W(user_id deleted_by_user_id),
+      literal_fields: ~W(author fingerprint deleted_by_user),
+      ngram_fields: fields[:ngram_fields] ++ ~W(deletion_reason),
       ip_fields: ~W(ip),
-      bool_fields: ~W(anonymous deleted),
+      bool_fields: ~W(anonymous deleted destroyed_content),
       custom_fields: fields[:custom_fields] -- ~W(author user_id),
-      aliases: Map.merge(fields[:aliases], %{"deleted" => "hidden_from_users"}),
-      transforms: Map.drop(fields[:transforms], ~W(author user_id))
+      transforms: Map.drop(fields[:transforms], ~W(author user_id)),
+      aliases: %{"deleted" => "hidden_from_users"}
     )
   end
 

lib/philomena/comments/search_index.ex

@@ -19,19 +19,22 @@ defmodule Philomena.Comments.SearchIndex do
         dynamic: false,
         properties: %{
           id: %{type: "integer"},
-          posted_at: %{type: "date"},
+          created_at: %{type: "date"},
+          updated_at: %{type: "date"},
           ip: %{type: "ip"},
           fingerprint: %{type: "keyword"},
           image_id: %{type: "keyword"},
           user_id: %{type: "keyword"},
           author: %{type: "keyword"},
           image_tag_ids: %{type: "keyword"},
-          # boolean
-          anonymous: %{type: "keyword"},
-          # boolean
-          hidden_from_users: %{type: "keyword"},
+          anonymous: %{type: "boolean"},
+          hidden_from_users: %{type: "boolean"},
           body: %{type: "text", analyzer: "snowball"},
-          approved: %{type: "boolean"}
+          approved: %{type: "boolean"},
+          deleted_by_user: %{type: "keyword"},
+          deleted_by_user_id: %{type: "keyword"},
+          deletion_reason: %{type: "text", analyzer: "snowball"},
+          destroyed_content: %{type: "boolean"}
         }
       }
     }
@@ -41,24 +44,42 @@ defmodule Philomena.Comments.SearchIndex do
   def as_json(comment) do
     %{
       id: comment.id,
-      posted_at: comment.created_at,
-      ip: comment.ip |> to_string,
+      created_at: comment.created_at,
+      updated_at: comment.updated_at,
+      ip: to_string(comment.ip),
       fingerprint: comment.fingerprint,
       image_id: comment.image_id,
       user_id: comment.user_id,
-      author: if(!!comment.user and !comment.anonymous, do: comment.user.name),
+      author: if(!!comment.user, do: String.downcase(comment.user.name)),
       image_tag_ids: comment.image.tags |> Enum.map(& &1.id),
       anonymous: comment.anonymous,
       hidden_from_users: comment.image.hidden_from_users || comment.hidden_from_users,
       body: comment.body,
-      approved: comment.image.approved && comment.approved
+      approved: comment.image.approved && comment.approved,
+      deleted_by_user: if(!!comment.deleted_by, do: String.downcase(comment.deleted_by.name)),
+      deleted_by_user_id: comment.deleted_by_id,
+      deletion_reason: comment.deletion_reason,
+      destroyed_content: comment.destroyed_content
     }
   end
 
   def user_name_update_by_query(old_name, new_name) do
+    old_name = String.downcase(old_name)
+    new_name = String.downcase(new_name)
+
     %{
-      query: %{term: %{author: old_name}},
-      replacements: [%{path: ["author"], old: old_name, new: new_name}],
+      query: %{
+        bool: %{
+          should: [
+            %{term: %{author: old_name}},
+            %{term: %{deleted_by_user: old_name}}
+          ]
+        }
+      },
+      replacements: [
+        %{path: ["author"], old: old_name, new: new_name},
+        %{path: ["deleted_by_user"], old: old_name, new: new_name}
+      ],
       set_replacements: []
     }
   end

lib/philomena/filters/search_index.ex

@@ -30,11 +30,8 @@ defmodule Philomena.Filters.SearchIndex do
           hidden_count: %{type: "integer"},
           spoilered_tag_ids: %{type: "keyword"},
           hidden_tag_ids: %{type: "keyword"},
-          spoilered_tags: %{type: "keyword"},
-          hidden_tags: %{type: "keyword"},
           spoilered_complex_str: %{type: "keyword"},
-          hidden_complex_str: %{type: "keyword"},
-          user_count: %{type: "integer"}
+          hidden_complex_str: %{type: "keyword"}
         }
       }
     }
@@ -58,8 +55,7 @@ defmodule Philomena.Filters.SearchIndex do
       spoilered_complex_str:
         if(!!filter.spoilered_complex_str, do: String.downcase(filter.spoilered_complex_str)),
       hidden_complex_str:
-        if(!!filter.hidden_complex_str, do: String.downcase(filter.hidden_complex_str)),
-      user_count: filter.user_count
+        if(!!filter.hidden_complex_str, do: String.downcase(filter.hidden_complex_str))
     }
   end
 

lib/philomena/galleries/query.ex

@@ -4,12 +4,13 @@ defmodule Philomena.Galleries.Query do
   defp fields do
     [
       int_fields: ~W(id image_count watcher_count),
-      numeric_fields: ~W(image_ids watcher_ids),
+      numeric_fields: ~W(user_id image_ids thumbnail_id),
       literal_fields: ~W(title user),
       date_fields: ~W(created_at updated_at),
-      ngram_fields: ~W(description),
+      ngram_fields: ~W(description spoiler_warning),
       default_field: {"title", :term},
       aliases: %{
+        "user_id" => "creator_id",
         "user" => "creator"
       }
     ]

lib/philomena/galleries/search_index.ex

@@ -18,19 +18,18 @@ defmodule Philomena.Galleries.SearchIndex do
       mappings: %{
         dynamic: false,
         properties: %{
-          # keyword
           id: %{type: "integer"},
           image_count: %{type: "integer"},
           watcher_count: %{type: "integer"},
           updated_at: %{type: "date"},
           created_at: %{type: "date"},
           title: %{type: "keyword"},
-          # missing creator_id
+          creator_id: %{type: "keyword"},
           creator: %{type: "keyword"},
           image_ids: %{type: "keyword"},
-          # ???
-          watcher_ids: %{type: "keyword"},
-          description: %{type: "text", analyzer: "snowball"}
+          description: %{type: "text", analyzer: "snowball"},
+          thumbnail_id: %{type: "keyword"},
+          spoiler_warning: %{type: "text", analyzer: "snowball"}
         }
       }
     }
@@ -42,13 +41,15 @@ defmodule Philomena.Galleries.SearchIndex do
       id: gallery.id,
       image_count: gallery.image_count,
       watcher_count: length(gallery.subscribers),
-      watcher_ids: Enum.map(gallery.subscribers, & &1.id),
       updated_at: gallery.updated_at,
       created_at: gallery.created_at,
       title: String.downcase(gallery.title),
+      creator_id: gallery.creator_id,
       creator: String.downcase(gallery.creator.name),
       image_ids: Enum.map(gallery.interactions, & &1.image_id),
-      description: gallery.description
+      description: gallery.description,
+      thumbnail_id: gallery.thumbnail_id,
+      spoiler_warning: gallery.spoiler_warning
     }
   end
 

lib/philomena/images/query.ex

@@ -95,14 +95,14 @@ defmodule Philomena.Images.Query do
       bool_fields: ~W(animated processed thumbnails_generated),
       ngram_fields: ~W(description),
       custom_fields: ~W(gallery_id),
-      default_field: {"namespaced_tags.name", :term},
+      default_field: {"tags", :term},
       transforms: %{"gallery_id" => &gallery_id_transform/2},
       aliases: %{
         "faved_by" => "favourited_by_users",
         "faved_by_id" => "favourited_by_user_ids"
       },
       no_downcase_fields: ~W(file_name),
-      normalizations: %{"namespaced_tags.name" => &Tag.clean_tag_name/1}
+      normalizations: %{"tags" => &Tag.clean_tag_name/1}
     ]
   end
 
@@ -125,7 +125,7 @@ defmodule Philomena.Images.Query do
       literal_fields:
         fields[:literal_fields] ++
           ~W(fingerprint upvoted_by downvoted_by true_uploader hidden_by deleted_by_user),
-      ngram_fields: fields[:ngram_fields] ++ ~W(deletion_reason),
+      ngram_fields: fields[:ngram_fields] ++ ~W(deletion_reason scratchpad),
       ip_fields: ~W(ip),
       bool_fields: fields[:bool_fields] ++ ~W(anonymous deleted),
       aliases:

lib/philomena/images/search_index.ex

@@ -22,7 +22,6 @@ defmodule Philomena.Images.SearchIndex do
           anonymous: %{type: "boolean"},
           aspect_ratio: %{type: "float"},
           comment_count: %{type: "integer"},
-          commenters: %{type: "keyword"},
           created_at: %{type: "date"},
           deleted_by_user: %{type: "keyword"},
           deleted_by_user_id: %{type: "keyword"},
@@ -39,12 +38,10 @@ defmodule Philomena.Images.SearchIndex do
           file_name: %{type: "keyword"},
           fingerprint: %{type: "keyword"},
           first_seen_at: %{type: "date"},
-          fps: %{type: "float"},
-          frames: %{type: "integer"},
           height: %{type: "integer"},
           hidden_by_user_ids: %{type: "keyword"},
           hidden_by_users: %{type: "keyword"},
-          hidden_from_users: %{type: "keyword"},
+          hidden_from_users: %{type: "boolean"},
           id: %{type: "integer"},
           ip: %{type: "ip"},
           mime_type: %{type: "keyword"},
@@ -60,7 +57,6 @@ defmodule Philomena.Images.SearchIndex do
           source_count: %{type: "integer"},
           tag_count: %{type: "integer"},
           tag_ids: %{type: "keyword"},
-          tags: %{type: "text", analyzer: "keyword"},
           thumbnails_generated: %{type: "boolean"},
           true_uploader: %{type: "keyword"},
           true_uploader_id: %{type: "keyword"},
@@ -80,15 +76,7 @@ defmodule Philomena.Images.SearchIndex do
               position: %{type: "integer"}
             }
           },
-          gallery_id: %{type: "keyword"},
-          gallery_position: %{type: "object", enabled: false},
-          namespaced_tags: %{
-            properties: %{
-              name: %{type: "keyword"},
-              name_in_namespace: %{type: "keyword"},
-              namespace: %{type: "keyword"}
-            }
-          },
+          tags: %{type: "keyword"},
           approved: %{type: "boolean"},
           error_tag_count: %{type: "integer"},
           rating_tag_count: %{type: "integer"},
@@ -99,7 +87,8 @@ defmodule Philomena.Images.SearchIndex do
           body_type_tag_count: %{type: "integer"},
           content_fanmade_tag_count: %{type: "integer"},
           content_official_tag_count: %{type: "integer"},
-          spoiler_tag_count: %{type: "integer"}
+          spoiler_tag_count: %{type: "integer"},
+          scratchpad: %{type: "text", analyzer: "snowball"}
         }
       }
     }
@@ -127,7 +116,7 @@ defmodule Philomena.Images.SearchIndex do
       created_at: image.created_at,
       updated_at: image.updated_at,
       first_seen_at: image.first_seen_at,
-      ip: image.ip |> to_string,
+      ip: to_string(image.ip),
       tag_ids: image.tags |> Enum.map(& &1.id),
       mime_type: image.image_mime_type,
       uploader: if(!!image.user and !image.anonymous, do: String.downcase(image.user.name)),
@@ -155,11 +144,7 @@ defmodule Philomena.Images.SearchIndex do
       duplicate_id: image.duplicate_id,
       galleries:
         image.gallery_interactions |> Enum.map(&%{id: &1.gallery_id, position: &1.position}),
-      namespaced_tags: %{
-        name: image.tags |> Enum.flat_map(&([&1] ++ &1.aliases)) |> Enum.map(& &1.name)
-      },
-      gallery_id: Enum.map(image.gallery_interactions, & &1.gallery_id),
-      gallery_position: Map.new(image.gallery_interactions, &{&1.gallery_id, &1.position}),
+      tags: image.tags |> Enum.flat_map(&([&1] ++ &1.aliases)) |> Enum.map(& &1.name),
       favourited_by_users: image.favers |> Enum.map(&String.downcase(&1.name)),
       hidden_by_users: image.hiders |> Enum.map(&String.downcase(&1.name)),
       upvoters: image.upvoters |> Enum.map(&String.downcase(&1.name)),
@@ -175,7 +160,8 @@ defmodule Philomena.Images.SearchIndex do
       body_type_tag_count: Enum.count(image.tags, &(&1.category == "body-type")),
       content_fanmade_tag_count: Enum.count(image.tags, &(&1.category == "content-fanmade")),
       content_official_tag_count: Enum.count(image.tags, &(&1.category == "content-official")),
-      spoiler_tag_count: Enum.count(image.tags, &(&1.category == "spoiler"))
+      spoiler_tag_count: Enum.count(image.tags, &(&1.category == "spoiler")),
+      scratchpad: image.scratchpad
     }
   end
 

lib/philomena/posts.ex

@@ -380,7 +380,8 @@ defmodule Philomena.Posts do
 
     [
       user: user_query,
-      topic: topic_query
+      topic: topic_query,
+      deleted_by: user_query
     ]
   end
 

lib/philomena/posts/post.ex

@@ -72,6 +72,7 @@ defmodule Philomena.Posts.Post do
   def unhide_changeset(post) do
     change(post)
     |> put_change(:hidden_from_users, false)
+    |> put_change(:deleted_by_id, nil)
     |> put_change(:deletion_reason, "")
   end