defmodule BDS.ImportExecution do @moduledoc false alias BDS.Media alias BDS.Metadata alias BDS.Posts alias BDS.Posts.Post alias BDS.Repo alias BDS.Tags def execute_import(project_id, report, opts \\ []) when is_binary(project_id) and is_map(report) do normalized_report = normalize_report(report) default_author = Keyword.get(opts, :default_author) || project_default_author(project_id) uploads_folder_path = Keyword.get(opts, :uploads_folder_path) on_progress = Keyword.get(opts, :on_progress, fn _phase, _current, _total, _detail -> :ok end) category_items = List.wrap(get_in(normalized_report, [:items, :categories])) tag_items = List.wrap(get_in(normalized_report, [:items, :tags])) category_mapping = build_taxonomy_mapping(category_items) tag_mapping = build_taxonomy_mapping(tag_items) post_items = normalized_report |> import_items(:posts) |> Enum.filter(&(Map.get(&1, :post_type, "post") == "post")) page_items = import_items(normalized_report, :pages) media_items = import_items(normalized_report, :media) taxonomy_total = length(category_items) + length(tag_items) result = %{ success: true, tags: %{created: 0, skipped: 0}, posts: %{imported: 0, skipped: 0, errors: 0}, media: %{imported: 0, skipped: 0, errors: 0}, pages: %{imported: 0, skipped: 0, errors: 0}, wp_id_to_post_id: %{}, errors: [] } started_at = System.monotonic_time(:millisecond) notify_progress(on_progress, "tags", 0, taxonomy_total, "creating_tags", started_at) result = execute_taxonomies(category_items, tag_items, project_id, result, on_progress, started_at) notify_progress(on_progress, "posts", 0, length(post_items), "importing_posts", started_at) result = execute_posts( post_items, project_id, default_author, tag_mapping, category_mapping, result, on_progress, :posts, started_at ) notify_progress(on_progress, "media", 0, length(media_items), "importing_media", started_at) result = execute_media( media_items, project_id, default_author, result, on_progress, uploads_folder_path, started_at ) notify_progress(on_progress, "pages", 0, length(page_items), "importing_pages", started_at) result = execute_posts( page_items, project_id, default_author, tag_mapping, category_mapping, result, on_progress, :pages, started_at ) notify_progress(on_progress, "complete", 1, 1, "import_complete", started_at) {:ok, result} rescue error -> {:error, %{message: Exception.message(error)}} end defp execute_taxonomies(category_items, tag_items, project_id, result, on_progress, started_at) do items = category_items ++ tag_items total = length(items) items |> Enum.with_index(1) |> Enum.reduce(result, fn {item, index}, acc -> cond do Map.get(item, :exists_in_project) || not is_nil(Map.get(item, :mapped_to)) -> notify_progress( on_progress, "tags", index, total, "skipped_tag:#{item.name}", started_at ) put_in(acc, [:tags, :skipped], acc.tags.skipped + 1) true -> case Tags.create_tag(%{project_id: project_id, name: item.name}) do {:ok, _tag} -> notify_progress( on_progress, "tags", index, total, "created_tag:#{item.name}", started_at ) put_in(acc, [:tags, :created], acc.tags.created + 1) {:error, _reason} -> notify_progress( on_progress, "tags", index, total, "skipped_tag:#{item.name}", started_at ) put_in(acc, [:tags, :skipped], acc.tags.skipped + 1) end end end) end defp execute_posts( items, project_id, default_author, tag_mapping, category_mapping, result, on_progress, bucket, started_at ) do total = length(items) phase = Atom.to_string(bucket) Enum.with_index(items, 1) |> Enum.reduce(result, fn {item, index}, acc -> notify_progress(on_progress, phase, index, total, "processing:#{item.title}", started_at) execute_post_item( project_id, maybe_apply_page_category(item, bucket), acc, bucket, default_author, tag_mapping, category_mapping ) end) end defp execute_media( items, project_id, default_author, result, on_progress, uploads_folder_path, started_at ) do total = length(items) items |> Enum.with_index(1) |> Enum.reduce(result, fn {item, index}, acc -> notify_progress( on_progress, "media", index, total, "processing:#{item.filename}", started_at ) cond do item.status == "missing" -> put_in(acc, [:media, :skipped], acc.media.skipped + 1) item.status in ["update", "content-duplicate", "duplicate"] -> put_in(acc, [:media, :skipped], acc.media.skipped + 1) item.status == "conflict" and resolve_conflict(item) == "ignore" -> put_in(acc, [:media, :skipped], acc.media.skipped + 1) true -> case import_media_item(project_id, item, default_author, uploads_folder_path, acc) do {:ok, _media} -> put_in(acc, [:media, :imported], acc.media.imported + 1) {:error, reason} -> acc |> put_in([:media, :errors], acc.media.errors + 1) |> Map.update!(:errors, &(&1 ++ [inspect(reason)])) |> Map.put(:success, false) end end end) end defp execute_post_item( project_id, item, result, bucket, default_author, tag_mapping, category_mapping ) do cond do item.status in ["update", "content-duplicate", "duplicate"] -> put_in(result, [bucket, :skipped], get_in(result, [bucket, :skipped]) + 1) item.status == "conflict" and resolve_conflict(item) == "ignore" -> put_in(result, [bucket, :skipped], get_in(result, [bucket, :skipped]) + 1) item.status == "conflict" and resolve_conflict(item) == "overwrite" -> case overwrite_post_item(item, default_author, tag_mapping, category_mapping) do {:ok, post} -> result |> put_in([bucket, :imported], get_in(result, [bucket, :imported]) + 1) |> track_wp_id(item, post) {:error, reason} -> result |> put_in([bucket, :errors], get_in(result, [bucket, :errors]) + 1) |> Map.update!(:errors, &(&1 ++ [inspect(reason)])) |> Map.put(:success, false) end true -> case create_post_item(project_id, item, default_author, tag_mapping, category_mapping) do {:ok, post} -> result |> put_in([bucket, :imported], get_in(result, [bucket, :imported]) + 1) |> track_wp_id(item, post) {:error, reason} -> result |> put_in([bucket, :errors], get_in(result, [bucket, :errors]) + 1) |> Map.update!(:errors, &(&1 ++ [inspect(reason)])) |> Map.put(:success, false) end end end defp create_post_item(project_id, item, default_author, tag_mapping, category_mapping) do attrs = post_create_attrs(project_id, item, default_author, tag_mapping, category_mapping) with {:ok, post} <- Posts.create_post(attrs), :ok <- prepare_created_post(post.id, item, tag_mapping, category_mapping), {:ok, published_post} <- maybe_publish(post.id, item) do {:ok, published_post} end end defp overwrite_post_item(item, default_author, tag_mapping, category_mapping) do case Repo.get(Post, item.existing_id) do nil -> {:error, :not_found} %Post{} = post -> Posts.update_post(post.id, %{ title: item.title, excerpt: item.excerpt, content: item.content_markdown, author: item.author || default_author, tags: resolve_taxonomy(item.tags, tag_mapping), categories: resolve_taxonomy(item.categories, category_mapping), checksum: item.content_checksum }) end end defp import_media_item(project_id, item, default_author, uploads_folder_path, result) do source_path = item.source_file || uploads_source_path(item.relative_path, uploads_folder_path) checksum = if(source_path != nil and File.exists?(source_path), do: md5(File.read!(source_path)), else: nil ) linked_post_ids = parent_post_ids(item, result) if source_path && File.exists?(source_path) do case {item.status, resolve_conflict(item)} do {"conflict", "overwrite"} when item.existing_id != nil -> with {:ok, _replaced} <- Media.replace_media_file(item.existing_id, source_path), {:ok, _updated_media} <- Media.update_media(item.existing_id, %{ title: item.title, alt: item.description, author: default_author }) do link_media(linked_post_ids, item.existing_id) {:ok, Repo.get!(Media.Media, item.existing_id)} end _ -> attrs = %{ project_id: project_id, source_path: source_path, title: item.title, alt: item.description, author: default_author, checksum: checksum } attrs = if linked_post_ids == [], do: attrs, else: Map.put(attrs, :linked_post_ids, linked_post_ids) case Media.import_media(attrs) do {:ok, %{id: media_id} = media} -> link_media(linked_post_ids, media_id) {:ok, media} other -> other end end else {:error, :missing_source_file} end end defp link_media([], _media_id), do: :ok defp link_media(post_ids, media_id) when is_list(post_ids) do Enum.each(post_ids, fn post_id -> try do Media.link_media_to_post(media_id, post_id) rescue _ -> :ok catch _, _ -> :ok end end) :ok end defp parent_post_ids(item, result) do case Map.get(item, :parent_wp_id) do nil -> [] 0 -> [] wp_id -> case Map.get(result.wp_id_to_post_id, wp_id) do nil -> [] post_id -> [post_id] end end end defp track_wp_id(result, %{wp_id: wp_id}, %{id: post_id}) when is_integer(wp_id) and not is_nil(post_id) do update_in(result, [:wp_id_to_post_id], &Map.put(&1, wp_id, post_id)) end defp track_wp_id(result, _item, _post), do: result defp maybe_publish(post_id, item) do case item.wp_status do "publish" -> Posts.publish_post(post_id) _other -> {:ok, Repo.get!(Post, post_id)} end end defp prepare_created_post(post_id, item, tag_mapping, category_mapping) do case Repo.get(Post, post_id) do nil -> {:error, :not_found} %Post{} = post -> desired_slug = desired_slug(post, item) created_at = parse_timestamp(item.created_at) || post.created_at updated_at = parse_timestamp(item.updated_at) || created_at published_at = parse_timestamp(item.published_at) || created_at post |> Post.changeset(%{ slug: desired_slug, title: item.title, excerpt: item.excerpt, content: item.content_markdown, author: item.author, tags: resolve_taxonomy(item.tags, tag_mapping), categories: resolve_taxonomy(item.categories, category_mapping), checksum: item.content_checksum, created_at: created_at, updated_at: updated_at, published_at: if(item.wp_status == "publish", do: published_at, else: nil) }) |> Repo.update() |> case do {:ok, _updated} -> :ok error -> error end end end defp desired_slug(post, item) do if item.status == "conflict" and resolve_conflict(item) == "import" do post.slug else item.slug || post.slug end end defp post_create_attrs(project_id, item, default_author, tag_mapping, category_mapping) do %{ project_id: project_id, title: item.title, excerpt: item.excerpt, content: item.content_markdown, author: item.author || default_author, tags: resolve_taxonomy(item.tags, tag_mapping), categories: resolve_taxonomy(item.categories, category_mapping), checksum: item.content_checksum } end defp maybe_apply_page_category(item, :pages) do categories = (Map.get(item, :categories) || []) |> Enum.uniq() |> Enum.concat(["page"]) |> Enum.uniq() %{item | categories: categories} end defp maybe_apply_page_category(item, _bucket), do: item defp build_taxonomy_mapping(items) do Enum.reduce(items, %{}, fn item, acc -> key = item.name |> to_string() |> String.downcase() resolved = cond do present_string?(Map.get(item, :mapped_to)) -> String.downcase(item.mapped_to) true -> key end Map.put(acc, key, %{ resolved: resolved, needs_creation: not item.exists_in_project and not present_string?(Map.get(item, :mapped_to)) }) end) end defp resolve_taxonomy(items, mapping) when is_list(items) do items |> Enum.map(fn item -> key = item |> to_string() |> String.downcase() case Map.get(mapping, key) do %{resolved: resolved} -> resolved _ -> key end end) |> Enum.uniq() end defp resolve_taxonomy(_items, _mapping), do: [] defp resolve_conflict(item) do raw = Map.get(item, :resolution) normalize_resolution(raw) end defp normalize_resolution("ignore"), do: "ignore" defp normalize_resolution("skip"), do: "ignore" defp normalize_resolution("overwrite"), do: "overwrite" defp normalize_resolution("merge"), do: "overwrite" defp normalize_resolution("import"), do: "import" defp normalize_resolution(_other), do: "ignore" defp import_items(report, bucket) do items = get_in(report, [:items, bucket]) || [] details = get_in(report, [:details, bucket]) || [] if details == [] do Enum.map(items, &normalize_item/1) else detail_index = details |> Enum.map(&normalize_item/1) |> Map.new(fn item -> {item_identity(item), item} end) Enum.map(items, fn item -> normalized_item = normalize_item(item) identity = item_identity(normalized_item) detail_item = Map.get(detail_index, identity, normalized_item) if Map.has_key?(normalized_item, :resolution) do %{detail_item | resolution: normalized_item.resolution} else detail_item end end) end end defp item_identity(%{item_type: "media", filename: filename}), do: {:media, filename} defp item_identity(%{item_type: item_type, slug: slug}), do: {item_type, slug} defp normalize_report(report) when is_map(report) do report |> Enum.map(fn {key, value} -> normalized_key = if(is_binary(key), do: String.to_atom(key), else: key) {normalized_key, normalize_report(value)} end) |> Map.new() end defp normalize_report(report) when is_list(report), do: Enum.map(report, &normalize_report/1) defp normalize_report(report), do: report defp normalize_item(item) do normalize_report(item) end defp parse_timestamp(nil), do: nil defp parse_timestamp(value) when is_integer(value), do: value defp parse_timestamp(value) when is_binary(value) do value |> String.replace(" ", "T") |> NaiveDateTime.from_iso8601() |> case do {:ok, naive} -> DateTime.from_naive!(naive, "Etc/UTC") |> DateTime.to_unix(:millisecond) _other -> nil end end defp parse_timestamp(_value), do: nil defp uploads_source_path(relative_path, uploads_folder_path) defp uploads_source_path(relative_path, uploads_folder_path) when is_binary(relative_path) and is_binary(uploads_folder_path) and uploads_folder_path != "" do Path.join(uploads_folder_path, relative_path) end defp uploads_source_path(_relative_path, _uploads_folder_path), do: nil defp notify_progress(callback, phase, current, total, detail, started_at) when is_function(callback, 4) do eta = compute_eta(current, total, started_at) try do callback.(phase, current, total, %{detail: detail, eta: eta}) rescue _error -> try do callback.(phase, current, total, detail) rescue _error -> :ok end end :ok end defp compute_eta(current, total, started_at) when is_integer(current) and is_integer(total) and current > 0 and total > 0 and current <= total do elapsed = System.monotonic_time(:millisecond) - started_at if current >= total, do: 0, else: trunc(elapsed / current * (total - current)) end defp compute_eta(_current, _total, _started_at), do: nil defp md5(binary) do :md5 |> :crypto.hash(binary) |> Base.encode16(case: :lower) end defp present_string?(value) when is_binary(value) and value != "", do: true defp present_string?(_value), do: false defp project_default_author(project_id) do {:ok, metadata} = Metadata.get_project_metadata(project_id) Map.get(metadata, :default_author) end end