Add blurhash support to backend

Signed-off-by: Thomas Citharel <tcit@tcit.fr>
This commit is contained in:
Thomas Citharel 2021-06-10 09:36:25 +02:00
parent 8ef718121c
commit a24e08a6de
No known key found for this signature in database
GPG Key ID: A061B9DDE0CA0773
17 changed files with 161 additions and 44 deletions

View File

@ -66,6 +66,7 @@ config :mobilizon, Mobilizon.Web.Upload,
uploader: Mobilizon.Web.Upload.Uploader.Local, uploader: Mobilizon.Web.Upload.Uploader.Local,
filters: [ filters: [
Mobilizon.Web.Upload.Filter.Dedupe, Mobilizon.Web.Upload.Filter.Dedupe,
Mobilizon.Web.Upload.Filter.AnalyzeMetadata,
Mobilizon.Web.Upload.Filter.Optimize Mobilizon.Web.Upload.Filter.Optimize
], ],
allow_list_mime_types: ["image/gif", "image/jpeg", "image/png", "image/webp"], allow_list_mime_types: ["image/gif", "image/jpeg", "image/png", "image/webp"],

View File

@ -352,18 +352,14 @@ defmodule Mobilizon.Federation.ActivityPub.Utils do
end end
def make_media_data(media) when is_map(media) do def make_media_data(media) when is_map(media) do
with {:ok, %{"url" => [%{"href" => url, "mediaType" => content_type}], "size" => size}} <- with {:ok, %{url: url} = uploaded} <-
Mobilizon.Web.Upload.store(media.file), Mobilizon.Web.Upload.store(media.file),
{:media_exists, nil} <- {:media_exists, Mobilizon.Medias.get_media_by_url(url)}, {:media_exists, nil} <- {:media_exists, Mobilizon.Medias.get_media_by_url(url)},
{:ok, %Media{file: _file} = media} <- {:ok, %Media{file: _file} = media} <-
Mobilizon.Medias.create_media(%{ Mobilizon.Medias.create_media(%{
"file" => %{ file: Map.take(uploaded, [:url, :name, :content_type, :size]),
"url" => url, metadata: Map.take(uploaded, [:width, :height, :blurhash]),
"name" => media.name, actor_id: media.actor_id
"content_type" => content_type,
"size" => size
},
"actor_id" => media.actor_id
}) do }) do
Converter.Media.model_to_as(media) Converter.Media.model_to_as(media)
else else

View File

@ -143,7 +143,7 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Actor do
when code in 200..299 <- RemoteMediaDownloaderClient.get(url), when code in 200..299 <- RemoteMediaDownloaderClient.get(url),
name <- name || Parser.get_filename_from_response(response_headers, url) || default_name, name <- name || Parser.get_filename_from_response(response_headers, url) || default_name,
{:ok, file} <- Upload.store(%{body: body, name: name}) do {:ok, file} <- Upload.store(%{body: body, name: name}) do
file Map.take(file, [:content_type, :name, :url, :size])
end end
end end
end end

View File

@ -40,17 +40,13 @@ defmodule Mobilizon.Federation.ActivityStream.Converter.Media do
) )
when is_binary(media_url) do when is_binary(media_url) do
with {:ok, %{body: body}} <- Tesla.get(media_url, opts: @http_options), with {:ok, %{body: body}} <- Tesla.get(media_url, opts: @http_options),
{:ok, %{name: name, url: url, content_type: content_type, size: size}} <- {:ok, %{url: url} = uploaded} <-
Upload.store(%{body: body, name: name}), Upload.store(%{body: body, name: name}),
{:media_exists, nil} <- {:media_exists, Medias.get_media_by_url(url)} do {:media_exists, nil} <- {:media_exists, Medias.get_media_by_url(url)} do
Medias.create_media(%{ Medias.create_media(%{
"file" => %{ file: Map.take(uploaded, [:url, :name, :content_type, :size]),
"url" => url, metadata: Map.take(uploaded, [:width, :height, :blurhash]),
"name" => name, actor_id: actor_id
"content_type" => content_type,
"size" => size
},
"actor_id" => actor_id
}) })
else else
{:media_exists, %MediaModel{file: _file} = media} -> {:media_exists, %MediaModel{file: _file} = media} ->

View File

@ -52,14 +52,17 @@ defmodule Mobilizon.GraphQL.API.Events do
defp process_picture(%{media_id: _picture_id} = args, _), do: args defp process_picture(%{media_id: _picture_id} = args, _), do: args
defp process_picture(%{media: media}, %Actor{id: actor_id}) do defp process_picture(%{media: media}, %Actor{id: actor_id}) do
%{ with uploaded when is_map(uploaded) <-
file:
media media
|> Map.get(:file) |> Map.get(:file)
|> Utils.make_media_data(description: Map.get(media, :name)), |> Utils.make_media_data(description: Map.get(media, :name)) do
%{
file: Map.take(uploaded, [:url, :name, :content_type, :size]),
metadata: Map.take(uploaded, [:width, :height, :blurhash]),
actor_id: actor_id actor_id: actor_id
} }
end end
end
@spec extract_pictures_from_event_body(map(), Actor.t()) :: map() @spec extract_pictures_from_event_body(map(), Actor.t()) :: map()
defp extract_pictures_from_event_body( defp extract_pictures_from_event_body(

View File

@ -47,7 +47,13 @@ defmodule Mobilizon.GraphQL.Resolvers.Media do
%{context: %{current_user: %User{} = user}} %{context: %{current_user: %User{} = user}}
) do ) do
with %Actor{id: actor_id} <- Users.get_actor_for_user(user), with %Actor{id: actor_id} <- Users.get_actor_for_user(user),
{:ok, %{name: _name, url: url, content_type: content_type, size: size}} <- {:ok,
%{
name: _name,
url: url,
content_type: content_type,
size: size
} = uploaded} <-
Mobilizon.Web.Upload.store(file), Mobilizon.Web.Upload.store(file),
args <- args <-
args args
@ -55,7 +61,11 @@ defmodule Mobilizon.GraphQL.Resolvers.Media do
|> Map.put(:size, size) |> Map.put(:size, size)
|> Map.put(:content_type, content_type), |> Map.put(:content_type, content_type),
{:ok, media = %Media{}} <- {:ok, media = %Media{}} <-
Medias.create_media(%{"file" => args, "actor_id" => actor_id}) do Medias.create_media(%{
file: args,
actor_id: actor_id,
metadata: Map.take(uploaded, [:width, :height, :blurhash])
}) do
{:ok, transform_media(media)} {:ok, transform_media(media)}
else else
{:error, :mime_type_not_allowed} -> {:error, :mime_type_not_allowed} ->
@ -124,13 +134,14 @@ defmodule Mobilizon.GraphQL.Resolvers.Media do
def user_size(_parent, _args, _resolution), do: {:error, :unauthenticated} def user_size(_parent, _args, _resolution), do: {:error, :unauthenticated}
@spec transform_media(Media.t()) :: map() @spec transform_media(Media.t()) :: map()
defp transform_media(%Media{id: id, file: file}) do defp transform_media(%Media{id: id, file: file, metadata: metadata}) do
%{ %{
name: file.name, name: file.name,
url: file.url, url: file.url,
id: id, id: id,
content_type: file.content_type, content_type: file.content_type,
size: file.size size: file.size,
metadata: metadata
} }
end end

View File

@ -215,14 +215,17 @@ defmodule Mobilizon.GraphQL.Resolvers.Post do
defp process_picture(%{media_id: _picture_id} = args, _), do: args defp process_picture(%{media_id: _picture_id} = args, _), do: args
defp process_picture(%{media: media}, %Actor{id: actor_id}) do defp process_picture(%{media: media}, %Actor{id: actor_id}) do
%{ with uploaded when is_map(uploaded) <-
file:
media media
|> Map.get(:file) |> Map.get(:file)
|> Utils.make_media_data(description: Map.get(media, :name)), |> Utils.make_media_data(description: Map.get(media, :name)) do
%{
file: Map.take(uploaded, [:url, :name, :content_type, :size]),
metadata: Map.take(uploaded, [:width, :height, :blurhash]),
actor_id: actor_id actor_id: actor_id
} }
end end
end
@spec extract_pictures_from_post_body(map(), String.t()) :: map() @spec extract_pictures_from_post_body(map(), String.t()) :: map()
defp extract_pictures_from_post_body(%{body: body} = args, actor_id) do defp extract_pictures_from_post_body(%{body: body} = args, actor_id) do

View File

@ -14,6 +14,7 @@ defmodule Mobilizon.GraphQL.Schema.MediaType do
field(:url, :string, description: "The media's full URL") field(:url, :string, description: "The media's full URL")
field(:content_type, :string, description: "The media's detected content type") field(:content_type, :string, description: "The media's detected content type")
field(:size, :integer, description: "The media's size") field(:size, :integer, description: "The media's size")
field(:metadata, :media_metadata, description: "The media's metadata")
end end
@desc """ @desc """
@ -24,6 +25,15 @@ defmodule Mobilizon.GraphQL.Schema.MediaType do
field(:total, :integer, description: "The total number of medias in the list") field(:total, :integer, description: "The total number of medias in the list")
end end
@desc """
Some metadata associated with a media
"""
object :media_metadata do
field(:width, :integer, description: "The media width (if a picture)")
field(:height, :integer, description: "The media width (if a height)")
field(:blurhash, :string, description: "The media blurhash (if a picture")
end
@desc "An attached media or a link to a media" @desc "An attached media or a link to a media"
input_object :media_input do input_object :media_input do
# Either a full media object # Either a full media object

View File

@ -80,11 +80,12 @@ defmodule Mobilizon.Discussions do
# However, it also excludes all top-level comments with deleted replies from being selected # However, it also excludes all top-level comments with deleted replies from being selected
# |> where([_, r], is_nil(r.deleted_at)) # |> where([_, r], is_nil(r.deleted_at))
|> group_by([c], c.id) |> group_by([c], c.id)
|> order_by([c], desc: :is_announcement, asc: :published_at)
|> select([c, r], %{c | total_replies: count(r.id)}) |> select([c, r], %{c | total_replies: count(r.id)})
end end
def query(Comment, _) do def query(Comment, _) do
order_by(Comment, [c], asc: :published_at) order_by(Comment, [c], asc: :is_announcement, asc: :published_at)
end end
def query(queryable, _) do def query(queryable, _) do

View File

@ -256,7 +256,7 @@ defmodule Mobilizon.Events.Event do
# In case it's a new picture # In case it's a new picture
defp put_picture(%Changeset{} = changeset, _attrs) do defp put_picture(%Changeset{} = changeset, _attrs) do
cast_assoc(changeset, :picture) cast_assoc(changeset, :picture, with: &Media.changeset/2)
end end
# Created or updated with draft parameter: don't publish # Created or updated with draft parameter: don't publish

View File

@ -5,21 +5,32 @@ defmodule Mobilizon.Medias.Media do
use Ecto.Schema use Ecto.Schema
import Ecto.Changeset, only: [cast: 3, cast_embed: 2] import Ecto.Changeset, only: [cast: 3, cast_embed: 2, cast_embed: 3]
alias Mobilizon.Actors.Actor alias Mobilizon.Actors.Actor
alias Mobilizon.Discussions.Comment alias Mobilizon.Discussions.Comment
alias Mobilizon.Events.Event alias Mobilizon.Events.Event
alias Mobilizon.Medias.File alias Mobilizon.Medias.File
alias Mobilizon.Medias.Media.Metadata
alias Mobilizon.Posts.Post alias Mobilizon.Posts.Post
@type t :: %__MODULE__{ @type t :: %__MODULE__{
file: File.t(), file: File.t(),
metadata: Metadata.t(),
actor: Actor.t() actor: Actor.t()
} }
@metadata_attrs [:height, :width, :blurhash]
schema "medias" do schema "medias" do
embeds_one(:file, File, on_replace: :update) embeds_one(:file, File, on_replace: :update)
embeds_one :metadata, Metadata, on_replace: :update do
field(:height, :integer)
field(:width, :integer)
field(:blurhash, :string)
end
belongs_to(:actor, Actor) belongs_to(:actor, Actor)
has_many(:event_picture, Event, foreign_key: :picture_id) has_many(:event_picture, Event, foreign_key: :picture_id)
many_to_many(:events, Event, join_through: "events_medias") many_to_many(:events, Event, join_through: "events_medias")
@ -36,5 +47,13 @@ defmodule Mobilizon.Medias.Media do
media media
|> cast(attrs, [:actor_id]) |> cast(attrs, [:actor_id])
|> cast_embed(:file) |> cast_embed(:file)
|> cast_embed(:metadata, with: &metadata_changeset/2)
end
@doc false
@spec changeset(struct(), map) :: Ecto.Changeset.t()
def metadata_changeset(metadata, attrs) do
metadata
|> cast(attrs, @metadata_attrs)
end end
end end

View File

@ -0,0 +1,47 @@
# Portions of this file are derived from Pleroma:
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
# Upstream: https://git.pleroma.social/pleroma/pleroma/-/blob/develop/lib/pleroma/upload/filter/analyze_metadata.ex
defmodule Mobilizon.Web.Upload.Filter.AnalyzeMetadata do
@moduledoc """
Extracts metadata about the upload, such as width/height
"""
require Logger
alias Mobilizon.Web.Upload
@behaviour Mobilizon.Web.Upload.Filter
@spec filter(Upload.t()) ::
{:ok, :filtered, Upload.t()} | {:ok, :noop} | {:error, String.t()}
def filter(%Upload{tempfile: file, content_type: "image" <> _} = upload) do
image =
file
|> Mogrify.open()
|> Mogrify.verbose()
upload =
upload
|> Map.put(:width, image.width)
|> Map.put(:height, image.height)
|> Map.put(:blurhash, get_blurhash(file))
{:ok, :filtered, upload}
rescue
e in ErlangError ->
Logger.warn("#{__MODULE__}: #{inspect(e)}")
{:ok, :noop}
end
def filter(_), do: {:ok, :noop}
defp get_blurhash(file) do
case :eblurhash.magick(to_charlist(file)) do
{:ok, blurhash} ->
to_string(blurhash)
_ ->
nil
end
end
end

View File

@ -73,12 +73,9 @@ defmodule Mobilizon.Web.Upload do
{:ok, upload} <- Filter.filter(opts.filters, upload), {:ok, upload} <- Filter.filter(opts.filters, upload),
{:ok, url_spec} <- Uploader.put_file(opts.uploader, upload) do {:ok, url_spec} <- Uploader.put_file(opts.uploader, upload) do
{:ok, {:ok,
%{ upload
name: Map.get(opts, :description) || upload.name, |> Map.put(:name, Map.get(opts, :description) || upload.name)
url: url_from_spec(upload, opts.base_url, url_spec), |> Map.put(:url, url_from_spec(upload, opts.base_url, url_spec))}
content_type: upload.content_type,
size: upload.size
}}
else else
{:error, error} -> {:error, error} ->
Logger.error( Logger.error(

View File

@ -162,6 +162,9 @@ defmodule Mobilizon.Mixfile do
{:sweet_xml, "~> 0.6.6"}, {:sweet_xml, "~> 0.6.6"},
{:web_push_encryption, {:web_push_encryption,
git: "https://github.com/tcitworld/elixir-web-push-encryption", branch: "otp-24"}, git: "https://github.com/tcitworld/elixir-web-push-encryption", branch: "otp-24"},
{:eblurhash,
git: "https://github.com/zotonic/eblurhash",
ref: "04a0b76eadf4de1be17726f39b6313b88708fd12"},
# Dev and test dependencies # Dev and test dependencies
{:phoenix_live_reload, "~> 1.2", only: [:dev, :e2e]}, {:phoenix_live_reload, "~> 1.2", only: [:dev, :e2e]},
{:ex_machina, "~> 2.3", only: [:dev, :test]}, {:ex_machina, "~> 2.3", only: [:dev, :test]},

View File

@ -25,6 +25,7 @@
"dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"}, "dialyxir": {:hex, :dialyxir, "1.1.0", "c5aab0d6e71e5522e77beff7ba9e08f8e02bad90dfbeffae60eaf0cb47e29488", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "07ea8e49c45f15264ebe6d5b93799d4dd56a44036cf42d0ad9c960bc266c0b9a"},
"earmark": {:hex, :earmark, "1.4.15", "2c7f924bf495ec1f65bd144b355d0949a05a254d0ec561740308a54946a67888", [:mix], [{:earmark_parser, ">= 1.4.13", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "3b1209b85bc9f3586f370f7c363f6533788fb4e51db23aa79565875e7f9999ee"}, "earmark": {:hex, :earmark, "1.4.15", "2c7f924bf495ec1f65bd144b355d0949a05a254d0ec561740308a54946a67888", [:mix], [{:earmark_parser, ">= 1.4.13", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "3b1209b85bc9f3586f370f7c363f6533788fb4e51db23aa79565875e7f9999ee"},
"earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"}, "earmark_parser": {:hex, :earmark_parser, "1.4.13", "0c98163e7d04a15feb62000e1a891489feb29f3d10cb57d4f845c405852bbef8", [:mix], [], "hexpm", "d602c26af3a0af43d2f2645613f65841657ad6efc9f0e361c3b6c06b578214ba"},
"eblurhash": {:git, "https://github.com/zotonic/eblurhash", "04a0b76eadf4de1be17726f39b6313b88708fd12", [ref: "04a0b76eadf4de1be17726f39b6313b88708fd12"]},
"ecto": {:hex, :ecto, "3.6.2", "efdf52acfc4ce29249bab5417415bd50abd62db7b0603b8bab0d7b996548c2bc", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "efad6dfb04e6f986b8a3047822b0f826d9affe8e4ebdd2aeedbfcb14fd48884e"}, "ecto": {:hex, :ecto, "3.6.2", "efdf52acfc4ce29249bab5417415bd50abd62db7b0603b8bab0d7b996548c2bc", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "efad6dfb04e6f986b8a3047822b0f826d9affe8e4ebdd2aeedbfcb14fd48884e"},
"ecto_autoslug_field": {:hex, :ecto_autoslug_field, "2.0.1", "2177c1c253f6dd3efd4b56d1cb76104d0a6ef044c6b9a7a0ad6d32665c4111e5", [:mix], [{:ecto, ">= 2.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:slugger, ">= 0.2.0", [hex: :slugger, repo: "hexpm", optional: false]}], "hexpm", "a3cc73211f2e75b89a03332183812ebe1ac08be2e25a1df5aa3d1422f92c45c3"}, "ecto_autoslug_field": {:hex, :ecto_autoslug_field, "2.0.1", "2177c1c253f6dd3efd4b56d1cb76104d0a6ef044c6b9a7a0ad6d32665c4111e5", [:mix], [{:ecto, ">= 2.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:slugger, ">= 0.2.0", [hex: :slugger, repo: "hexpm", optional: false]}], "hexpm", "a3cc73211f2e75b89a03332183812ebe1ac08be2e25a1df5aa3d1422f92c45c3"},
"ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},

View File

@ -0,0 +1,9 @@
defmodule Mobilizon.Storage.Repo.Migrations.AddMetadataToMedia do
use Ecto.Migration
def change do
alter table(:medias) do
add(:metadata, :map)
end
end
end

View File

@ -0,0 +1,20 @@
# Portions of this file are derived from Pleroma:
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
# Upstream: https://git.pleroma.social/pleroma/pleroma/-/blob/develop/test/pleroma/upload/filter/analyze_metadata_test.exs
defmodule Mobilizon.Web.Upload.Filter.AnalyzeMetadataTest do
use Mobilizon.DataCase, async: true
alias Mobilizon.Web.Upload.Filter.AnalyzeMetadata
test "adds the image dimensions" do
upload = %Mobilizon.Web.Upload{
name: "an… image.jpg",
content_type: "image/jpeg",
path: Path.absname("test/fixtures/image.jpg"),
tempfile: Path.absname("test/fixtures/image.jpg")
}
assert {:ok, :filtered, %{width: 266, height: 67}} = AnalyzeMetadata.filter(upload)
end
end