Parallel image upload in dev seeds

Author: MareStare

lib/philomena_media/objects.ex

@@ -61,11 +61,20 @@ defmodule PhilomenaMedia.Objects do
     contents =
       backends()
       |> Enum.find_value(fn opts ->
-        ExAws.S3.get_object(opts[:bucket], key)
+        bucket = opts[:bucket]
+
+        ExAws.S3.get_object(bucket, key)
         |> ExAws.request(opts[:config_overrides])
         |> case do
-          {:ok, result} -> result
-          _ -> nil
+          {:ok, result} ->
+            result
+
+          {:err, err} ->
+            Logger.warning(
+              "Failed to download #{key} from #{bucket}: #{inspect(err, pretty: true)}"
+            )
+
+            nil
         end
       end)
 

priv/repo/seeds_development.exs

@@ -19,6 +19,7 @@ defmodule Philomena.DevSeeds do
   alias Philomena.Posts
   alias Philomena.Tags
   import Ecto.Query
+  require Logger
 
   def seed() do
     {:ok, _} = Application.ensure_all_started(:plug)
@@ -28,11 +29,6 @@ defmodule Philomena.DevSeeds do
       |> File.read!()
       |> Jason.decode!()
 
-    images =
-      "priv/repo/seeds/dev/images.json"
-      |> File.read!()
-      |> Jason.decode!()
-
     # pages =
     #   "priv/repo/seeds/dev/pages.json"
     #   |> File.read!()
@@ -63,39 +59,7 @@ defmodule Philomena.DevSeeds do
 
     IO.puts("---- Generating images")
 
-    for image_def <- images do
-      file = Briefly.create!()
-      now = DateTime.utc_now() |> DateTime.to_unix(:microsecond)
-
-      IO.puts("Fetching #{image_def["url"]} ...")
-      {:ok, %{body: body}} = PhilomenaProxy.Http.get(image_def["url"])
-
-      File.write!(file, body)
-
-      upload = %Plug.Upload{
-        path: file,
-        content_type: "application/octet-stream",
-        filename: "fixtures-#{now}"
-      }
-
-      IO.puts("Inserting ...")
-
-      Images.create_image(
-        pleb_attrs,
-        Map.merge(image_def, %{"image" => upload})
-      )
-      |> case do
-        {:ok, %{image: image}} ->
-          Images.approve_image(image)
-          Images.reindex_image(image)
-          Tags.reindex_tags(image.added_tags)
-
-          IO.puts("Created image ##{image.id}")
-
-        {:error, :image, changeset, _so_far} ->
-          IO.inspect(changeset.errors)
-      end
-    end
+    generate_images(pleb_attrs)
 
     IO.puts("---- Generating comments for image #1")
 
@@ -157,6 +121,48 @@ defmodule Philomena.DevSeeds do
     Logger.configure(level: :debug)
   end
 
+  defp generate_images(pleb_attrs) do
+    images =
+      "priv/repo/seeds/dev/images.json"
+      |> File.read!()
+      |> Jason.decode!()
+
+    ingest_image = fn image_def ->
+      file = Briefly.create!()
+      now = DateTime.utc_now() |> DateTime.to_unix(:microsecond)
+
+      IO.puts("Fetching #{image_def["url"]} ...")
+      {:ok, %{body: body}} = PhilomenaProxy.Http.get(image_def["url"])
+
+      File.write!(file, body)
+
+      upload = %Plug.Upload{
+        path: file,
+        content_type: "application/octet-stream",
+        filename: "fixtures-#{now}"
+      }
+
+      IO.puts("Inserting ...")
+
+      Images.create_image(pleb_attrs, Map.merge(image_def, %{"image" => upload}))
+      |> case do
+        {:ok, %{image: image}} ->
+          Images.approve_image(image)
+          Images.reindex_image(image)
+          Tags.reindex_tags(image.added_tags)
+
+          IO.puts("Created image ##{image.id}")
+
+        {:error, :image, changeset, _so_far} ->
+          IO.inspect(changeset.errors)
+      end
+    end
+
+    images
+    |> Task.async_stream(ingest_image, max_concurrency: 20, ordered: false)
+    |> Stream.run()
+  end
+
   defp default_ip() do
     {:ok, ip} = EctoNetwork.INET.cast({203, 0, 113, 0})
     ip