Add blurhash support to backend

Signed-off-by: Thomas Citharel <tcit@tcit.fr>

config/config.exs

@@ -66,6 +66,7 @@ config :mobilizon, Mobilizon.Web.Upload,
   uploader: Mobilizon.Web.Upload.Uploader.Local,
   filters: [
     Mobilizon.Web.Upload.Filter.Dedupe,
+    Mobilizon.Web.Upload.Filter.AnalyzeMetadata,
     Mobilizon.Web.Upload.Filter.Optimize
   ],
   allow_list_mime_types: ["image/gif", "image/jpeg", "image/png", "image/webp"],

lib/federation/activity_pub/utils.ex

@@ -352,18 +352,14 @@ defmodule Mobilizon.Federation.ActivityPub.Utils do
   end
 
   def make_media_data(media) when is_map(media) do
-    with {:ok, %{"url" => [%{"href" => url, "mediaType" => content_type}], "size" => size}} <-
+    with {:ok, %{url: url} = uploaded} <-
            Mobilizon.Web.Upload.store(media.file),
          {:media_exists, nil} <- {:media_exists, Mobilizon.Medias.get_media_by_url(url)},
          {:ok, %Media{file: _file} = media} <-
            Mobilizon.Medias.create_media(%{
-             "file" => %{
-               "url" => url,
-               "name" => media.name,
-               "content_type" => content_type,
-               "size" => size
-             },
-             "actor_id" => media.actor_id
+             file: Map.take(uploaded, [:url, :name, :content_type, :size]),
+             metadata: Map.take(uploaded, [:width, :height, :blurhash]),
+             actor_id: media.actor_id
            }) do
       Converter.Media.model_to_as(media)
     else

lib/federation/activity_stream/converter/actor.ex

@@ -143,7 +143,7 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
          when code in 200..299 <- RemoteMediaDownloaderClient.get(url),
          name <- name || Parser.get_filename_from_response(response_headers, url) || default_name,
          {:ok, file} <- Upload.store(%{body: body, name: name}) do
-      file
+      Map.take(file, [:content_type, :name, :url, :size])
     end
   end
 end

lib/federation/activity_stream/converter/media.ex

@@ -40,17 +40,13 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Media do
       )
       when is_binary(media_url) do
     with {:ok, %{body: body}} <- Tesla.get(media_url, opts: @http_options),
-         {:ok, %{name: name, url: url, content_type: content_type, size: size}} <-
+         {:ok, %{url: url} = uploaded} <-
            Upload.store(%{body: body, name: name}),
          {:media_exists, nil} <- {:media_exists, Medias.get_media_by_url(url)} do
       Medias.create_media(%{
-        "file" => %{
-          "url" => url,
-          "name" => name,
-          "content_type" => content_type,
-          "size" => size
-        },
-        "actor_id" => actor_id
+        file: Map.take(uploaded, [:url, :name, :content_type, :size]),
+        metadata: Map.take(uploaded, [:width, :height, :blurhash]),
+        actor_id: actor_id
       })
     else
       {:media_exists, %MediaModel{file: _file} = media} ->

lib/graphql/api/events.ex

@@ -52,13 +52,16 @@ defmodule Mobilizon.GraphQL.API.Events do
   defp process_picture(%{media_id: _picture_id} = args, _), do: args
 
   defp process_picture(%{media: media}, %Actor{id: actor_id}) do
-    %{
-      file:
-        media
-        |> Map.get(:file)
-        |> Utils.make_media_data(description: Map.get(media, :name)),
-      actor_id: actor_id
-    }
+    with uploaded when is_map(uploaded) <-
+           media
+           |> Map.get(:file)
+           |> Utils.make_media_data(description: Map.get(media, :name)) do
+      %{
+        file: Map.take(uploaded, [:url, :name, :content_type, :size]),
+        metadata: Map.take(uploaded, [:width, :height, :blurhash]),
+        actor_id: actor_id
+      }
+    end
   end
 
   @spec extract_pictures_from_event_body(map(), Actor.t()) :: map()

lib/graphql/resolvers/media.ex

@@ -47,7 +47,13 @@ defmodule Mobilizon.GraphQL.Resolvers.Media do
         %{context: %{current_user: %User{} = user}}
       ) do
     with %Actor{id: actor_id} <- Users.get_actor_for_user(user),
-         {:ok, %{name: _name, url: url, content_type: content_type, size: size}} <-
+         {:ok,
+          %{
+            name: _name,
+            url: url,
+            content_type: content_type,
+            size: size
+          } = uploaded} <-
            Mobilizon.Web.Upload.store(file),
          args <-
            args
@@ -55,7 +61,11 @@ defmodule Mobilizon.GraphQL.Resolvers.Media do
            |> Map.put(:size, size)
            |> Map.put(:content_type, content_type),
          {:ok, media = %Media{}} <-
-           Medias.create_media(%{"file" => args, "actor_id" => actor_id}) do
+           Medias.create_media(%{
+             file: args,
+             actor_id: actor_id,
+             metadata: Map.take(uploaded, [:width, :height, :blurhash])
+           }) do
       {:ok, transform_media(media)}
     else
       {:error, :mime_type_not_allowed} ->
@@ -124,13 +134,14 @@ defmodule Mobilizon.GraphQL.Resolvers.Media do
   def user_size(_parent, _args, _resolution), do: {:error, :unauthenticated}
 
   @spec transform_media(Media.t()) :: map()
-  defp transform_media(%Media{id: id, file: file}) do
+  defp transform_media(%Media{id: id, file: file, metadata: metadata}) do
     %{
       name: file.name,
       url: file.url,
       id: id,
       content_type: file.content_type,
-      size: file.size
+      size: file.size,
+      metadata: metadata
     }
   end
 

lib/graphql/resolvers/post.ex

@@ -215,13 +215,16 @@ defmodule Mobilizon.GraphQL.Resolvers.Post do
   defp process_picture(%{media_id: _picture_id} = args, _), do: args
 
   defp process_picture(%{media: media}, %Actor{id: actor_id}) do
-    %{
-      file:
-        media
-        |> Map.get(:file)
-        |> Utils.make_media_data(description: Map.get(media, :name)),
-      actor_id: actor_id
-    }
+    with uploaded when is_map(uploaded) <-
+           media
+           |> Map.get(:file)
+           |> Utils.make_media_data(description: Map.get(media, :name)) do
+      %{
+        file: Map.take(uploaded, [:url, :name, :content_type, :size]),
+        metadata: Map.take(uploaded, [:width, :height, :blurhash]),
+        actor_id: actor_id
+      }
+    end
   end
 
   @spec extract_pictures_from_post_body(map(), String.t()) :: map()

lib/graphql/schema/media.ex

@@ -14,6 +14,7 @@ defmodule Mobilizon.GraphQL.Schema.MediaType do
     field(:url, :string, description: "The media's full URL")
     field(:content_type, :string, description: "The media's detected content type")
     field(:size, :integer, description: "The media's size")
+    field(:metadata, :media_metadata, description: "The media's metadata")
   end
 
   @desc """
@@ -24,6 +25,15 @@ defmodule Mobilizon.GraphQL.Schema.MediaType do
     field(:total, :integer, description: "The total number of medias in the list")
   end
 
+  @desc """
+  Some metadata associated with a media
+  """
+  object :media_metadata do
+    field(:width, :integer, description: "The media width (if a picture)")
+    field(:height, :integer, description: "The media width (if a height)")
+    field(:blurhash, :string, description: "The media blurhash (if a picture")
+  end
+
   @desc "An attached media or a link to a media"
   input_object :media_input do
     # Either a full media object

lib/mobilizon/discussions/discussions.ex

@@ -80,11 +80,12 @@ defmodule Mobilizon.Discussions do
     # However, it also excludes all top-level comments with deleted replies from being selected
     # |> where([_, r], is_nil(r.deleted_at))
     |> group_by([c], c.id)
+    |> order_by([c], desc: :is_announcement, asc: :published_at)
     |> select([c, r], %{c | total_replies: count(r.id)})
   end
 
   def query(Comment, _) do
-    order_by(Comment, [c], asc: :published_at)
+    order_by(Comment, [c], asc: :is_announcement, asc: :published_at)
   end
 
   def query(queryable, _) do

lib/mobilizon/events/event.ex

@@ -256,7 +256,7 @@ defmodule Mobilizon.Events.Event do
 
   # In case it's a new picture
   defp put_picture(%Changeset{} = changeset, _attrs) do
-    cast_assoc(changeset, :picture)
+    cast_assoc(changeset, :picture, with: &Media.changeset/2)
   end
 
   # Created or updated with draft parameter: don't publish

lib/mobilizon/medias/media.ex

@@ -5,21 +5,32 @@ defmodule Mobilizon.Medias.Media do
 
   use Ecto.Schema
 
-  import Ecto.Changeset, only: [cast: 3, cast_embed: 2]
+  import Ecto.Changeset, only: [cast: 3, cast_embed: 2, cast_embed: 3]
 
   alias Mobilizon.Actors.Actor
   alias Mobilizon.Discussions.Comment
   alias Mobilizon.Events.Event
   alias Mobilizon.Medias.File
+  alias Mobilizon.Medias.Media.Metadata
   alias Mobilizon.Posts.Post
 
   @type t :: %__MODULE__{
           file: File.t(),
+          metadata: Metadata.t(),
           actor: Actor.t()
         }
 
+  @metadata_attrs [:height, :width, :blurhash]
+
   schema "medias" do
     embeds_one(:file, File, on_replace: :update)
+
+    embeds_one :metadata, Metadata, on_replace: :update do
+      field(:height, :integer)
+      field(:width, :integer)
+      field(:blurhash, :string)
+    end
+
     belongs_to(:actor, Actor)
     has_many(:event_picture, Event, foreign_key: :picture_id)
     many_to_many(:events, Event, join_through: "events_medias")
@@ -36,5 +47,13 @@ defmodule Mobilizon.Medias.Media do
     media
     |> cast(attrs, [:actor_id])
     |> cast_embed(:file)
+    |> cast_embed(:metadata, with: &metadata_changeset/2)
+  end
+
+  @doc false
+  @spec changeset(struct(), map) :: Ecto.Changeset.t()
+  def metadata_changeset(metadata, attrs) do
+    metadata
+    |> cast(attrs, @metadata_attrs)
   end
 end

lib/web/upload/filter/analyze_metadata.ex

@@ -0,0 +1,47 @@
+# Portions of this file are derived from Pleroma:
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+# Upstream: https://git.pleroma.social/pleroma/pleroma/-/blob/develop/lib/pleroma/upload/filter/analyze_metadata.ex
+
+defmodule Mobilizon.Web.Upload.Filter.AnalyzeMetadata do
+  @moduledoc """
+  Extracts metadata about the upload, such as width/height
+  """
+  require Logger
+  alias Mobilizon.Web.Upload
+
+  @behaviour Mobilizon.Web.Upload.Filter
+
+  @spec filter(Upload.t()) ::
+          {:ok, :filtered, Upload.t()} | {:ok, :noop} | {:error, String.t()}
+  def filter(%Upload{tempfile: file, content_type: "image" <> _} = upload) do
+    image =
+      file
+      |> Mogrify.open()
+      |> Mogrify.verbose()
+
+    upload =
+      upload
+      |> Map.put(:width, image.width)
+      |> Map.put(:height, image.height)
+      |> Map.put(:blurhash, get_blurhash(file))
+
+    {:ok, :filtered, upload}
+  rescue
+    e in ErlangError ->
+      Logger.warn("#{__MODULE__}: #{inspect(e)}")
+      {:ok, :noop}
+  end
+
+  def filter(_), do: {:ok, :noop}
+
+  defp get_blurhash(file) do
+    case :eblurhash.magick(to_charlist(file)) do
+      {:ok, blurhash} ->
+        to_string(blurhash)
+
+      _ ->
+        nil
+    end
+  end
+end

lib/web/upload/upload.ex

@@ -73,12 +73,9 @@ defmodule Mobilizon.Web.Upload do
          {:ok, upload} <- Filter.filter(opts.filters, upload),
          {:ok, url_spec} <- Uploader.put_file(opts.uploader, upload) do
       {:ok,
-       %{
-         name: Map.get(opts, :description) || upload.name,
-         url: url_from_spec(upload, opts.base_url, url_spec),
-         content_type: upload.content_type,
-         size: upload.size
-       }}
+       upload
+       |> Map.put(:name, Map.get(opts, :description) || upload.name)
+       |> Map.put(:url, url_from_spec(upload, opts.base_url, url_spec))}
     else
       {:error, error} ->
         Logger.error(

mix.exs

@@ -162,6 +162,9 @@ defmodule Mobilizon.Mixfile do
       {:sweet_xml, "~> 0.6.6"},
       {:web_push_encryption,
        git: "https://github.com/tcitworld/elixir-web-push-encryption", branch: "otp-24"},
+      {:eblurhash,
+       git: "https://github.com/zotonic/eblurhash",
+       ref: "04a0b76eadf4de1be17726f39b6313b88708fd12"},
       # Dev and test dependencies
       {:phoenix_live_reload, "~> 1.2", only: [:dev, :e2e]},
       {:ex_machina, "~> 2.3", only: [:dev, :test]},

mix.lock

@@ -25,6 +25,7 @@
   "dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"},
   "earmark": {:hex, :earmark, "1.4.15", "2c7f924bf495ec1f65bd144b355d0949a05a254d0ec561740308a54946a67888", [:mix], [{:earmark_parser, ">= 1.4.13", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "3b1209b85bc9f3586f370f7c363f6533788fb4e51db23aa79565875e7f9999ee"},
   "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"},
+  "eblurhash": {:git, "https://github.com/zotonic/eblurhash", "04a0b76eadf4de1be17726f39b6313b88708fd12", [ref: "04a0b76eadf4de1be17726f39b6313b88708fd12"]},
   "ecto": {:hex, :ecto, "3.6.2", "efdf52acfc4ce29249bab5417415bd50abd62db7b0603b8bab0d7b996548c2bc", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "efad6dfb04e6f986b8a3047822b0f826d9affe8e4ebdd2aeedbfcb14fd48884e"},
   "ecto_autoslug_field": {:hex, :ecto_autoslug_field, "2.0.1", "2177c1c253f6dd3efd4b56d1cb76104d0a6ef044c6b9a7a0ad6d32665c4111e5", [:mix], [{:ecto, ">= 2.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:slugger, ">= 0.2.0", [hex: :slugger, repo: "hexpm", optional: false]}], "hexpm", "a3cc73211f2e75b89a03332183812ebe1ac08be2e25a1df5aa3d1422f92c45c3"},
   "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},

priv/repo/migrations/20210608133040_add_metadata_to_media.exs

@@ -0,0 +1,9 @@
+defmodule Mobilizon.Storage.Repo.Migrations.AddMetadataToMedia do
+  use Ecto.Migration
+
+  def change do
+    alter table(:medias) do
+      add(:metadata, :map)
+    end
+  end
+end

test/web/upload/filter/analyze_metadata_test.exs

@@ -0,0 +1,20 @@
+# Portions of this file are derived from Pleroma:
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+# Upstream: https://git.pleroma.social/pleroma/pleroma/-/blob/develop/test/pleroma/upload/filter/analyze_metadata_test.exs
+
+defmodule Mobilizon.Web.Upload.Filter.AnalyzeMetadataTest do
+  use Mobilizon.DataCase, async: true
+  alias Mobilizon.Web.Upload.Filter.AnalyzeMetadata
+
+  test "adds the image dimensions" do
+    upload = %Mobilizon.Web.Upload{
+      name: "an… image.jpg",
+      content_type: "image/jpeg",
+      path: Path.absname("test/fixtures/image.jpg"),
+      tempfile: Path.absname("test/fixtures/image.jpg")
+    }
+
+    assert {:ok, :filtered, %{width: 266, height: 67}} = AnalyzeMetadata.filter(upload)
+  end
+end