feat: more stuff on publishing

lib/bds/application.ex

@@ -8,6 +8,8 @@ defmodule BDS.Application do
     children = [
       BDS.Repo,
       BDS.Tasks,
+      BDS.Preview,
+      BDS.Publishing,
       {Task.Supervisor, name: BDS.Tasks.TaskSupervisor},
       BDS.Scripting.JobStore,
       {Task.Supervisor, name: BDS.Scripting.TaskSupervisor},

lib/bds/generation.ex

@@ -4,9 +4,63 @@ defmodule BDS.Generation do
   import Ecto.Query
 
   alias BDS.Generation.GeneratedFileHash
+  alias BDS.Metadata
+  alias BDS.Posts.Post
+  alias BDS.Posts.Translation
   alias BDS.Projects
   alias BDS.Repo
 
+  @core_sections [:core, :single]
+
+  def plan_generation(project_id, sections \\ [:core]) when is_binary(project_id) and is_list(sections) do
+    project = Projects.get_project!(project_id)
+    {:ok, metadata} = Metadata.get_project_metadata(project_id)
+    {:ok, generated_files} = list_generated_files(project_id)
+
+    {:ok,
+     %{
+       project_id: project_id,
+       project_name: project.name,
+       base_url: normalize_base_url(metadata.public_url),
+       language: metadata.main_language,
+       blog_languages: normalize_blog_languages(metadata.main_language, metadata.blog_languages),
+       max_posts_per_page: metadata.max_posts_per_page,
+       pico_theme: metadata.pico_theme,
+       sections: normalize_sections(sections),
+       generated_files: generated_files
+     }}
+  end
+
+  def generate_site(project_id, sections \\ [:core]) when is_binary(project_id) and is_list(sections) do
+    with {:ok, plan} <- plan_generation(project_id, sections) do
+      outputs = build_outputs(plan)
+
+      Enum.each(outputs, fn {relative_path, content} ->
+        {:ok, _write} = write_generated_file(project_id, relative_path, content)
+      end)
+
+      {:ok, generated_files} = list_generated_files(project_id)
+      {:ok, %{sections: plan.sections, generated_files: generated_files}}
+    end
+  end
+
+  def post_output_path(%Post{} = post), do: post_output_path(post, nil)
+
+  def post_output_path(%Post{} = post, language) do
+    datetime = DateTime.from_unix!(post.created_at)
+    year = Integer.to_string(datetime.year)
+    month = datetime.month |> Integer.to_string() |> String.pad_leading(2, "0")
+    day = datetime.day |> Integer.to_string() |> String.pad_leading(2, "0")
+
+    path_parts = [year, month, day, post.slug, "index.html"]
+
+    case language do
+      nil -> Path.join(path_parts)
+      "" -> Path.join(path_parts)
+      value -> Path.join([value | path_parts])
+    end
+  end
+
   def write_generated_file(project_id, relative_path, content)
       when is_binary(project_id) and is_binary(relative_path) and is_binary(content) do
     project = Projects.get_project!(project_id)
@@ -66,6 +120,228 @@ defmodule BDS.Generation do
     :ok
   end
 
+  defp build_outputs(plan) do
+    published_posts = list_published_posts(plan.project_id)
+    published_translations = list_published_translations(plan.project_id)
+    post_by_id = Map.new(published_posts, &{&1.id, &1})
+
+    core_outputs =
+      if :core in plan.sections do
+        build_core_outputs(plan, published_posts)
+      else
+        []
+      end
+
+    single_outputs =
+      if :single in plan.sections do
+        build_single_outputs(plan.project_id, published_posts, published_translations, post_by_id)
+      else
+        []
+      end
+
+    urls =
+      core_outputs ++ single_outputs
+      |> Enum.map(fn {relative_path, _content} -> url_for_output(plan.base_url, relative_path) end)
+
+    sitemap =
+      if :core in plan.sections do
+        [{"sitemap.xml", render_sitemap(urls)}]
+      else
+        []
+      end
+
+    core_outputs ++ single_outputs ++ sitemap
+  end
+
+  defp build_core_outputs(plan, published_posts) do
+    language = plan.language
+    additional_languages = Enum.reject(plan.blog_languages, &(&1 == language))
+
+    [
+      {"index.html", render_home(plan, language)},
+      {"feed.xml", render_feed(plan, language, published_posts)},
+      {"atom.xml", render_atom(plan, language, published_posts)},
+      {"calendar.json", render_calendar(published_posts)}
+    ] ++
+      Enum.flat_map(additional_languages, fn localized_language ->
+        [
+          {Path.join(localized_language, "index.html"), render_home(plan, localized_language)},
+          {Path.join(localized_language, "feed.xml"), render_feed(plan, localized_language, published_posts)},
+          {Path.join(localized_language, "atom.xml"), render_atom(plan, localized_language, published_posts)}
+        ]
+      end)
+  end
+
+  defp build_single_outputs(project_id, published_posts, published_translations, post_by_id) do
+    post_outputs =
+      Enum.map(published_posts, fn post ->
+        {post_output_path(post), render_post_page(post.title, load_body(project_id, post.file_path, post.content), post.slug, post.language)}
+      end)
+
+    translation_outputs =
+      Enum.flat_map(published_translations, fn translation ->
+        case post_by_id[translation.translation_for] do
+          nil ->
+            []
+
+          post ->
+            [
+              {post_output_path(post, translation.language),
+               render_post_page(translation.title, load_body(project_id, translation.file_path, translation.content), post.slug, translation.language)}
+            ]
+        end
+      end)
+
+    post_outputs ++ translation_outputs
+  end
+
+  defp list_published_posts(project_id) do
+    Repo.all(
+      from post in Post,
+        where: post.project_id == ^project_id and post.status == :published,
+        order_by: [asc: post.created_at, asc: post.slug]
+    )
+  end
+
+  defp list_published_translations(project_id) do
+    Repo.all(
+      from translation in Translation,
+        where: translation.project_id == ^project_id and translation.status == :published,
+        order_by: [asc: translation.created_at, asc: translation.language]
+    )
+  end
+
+  defp normalize_sections(sections) do
+    sections
+    |> Enum.filter(&(&1 in @core_sections))
+    |> Enum.uniq()
+    |> case do
+      [] -> [:core]
+      values -> values
+    end
+  end
+
+  defp normalize_base_url(nil), do: nil
+  defp normalize_base_url(url), do: String.trim_trailing(url, "/")
+
+  defp normalize_blog_languages(main_language, blog_languages) do
+    ([main_language] ++ (blog_languages || []))
+    |> Enum.reject(&(&1 in [nil, ""]))
+    |> Enum.uniq()
+  end
+
+  defp render_home(plan, language) do
+    [
+      "<html>",
+      "<head><title>",
+      plan.project_name,
+      "</title></head>",
+      "<body data-language=\"",
+      to_string(language),
+      "\"><main><h1>",
+      plan.project_name,
+      "</h1></main></body>",
+      "</html>"
+    ]
+    |> IO.iodata_to_binary()
+  end
+
+  defp render_feed(plan, language, published_posts) do
+    items =
+      published_posts
+      |> Enum.filter(&(&1.language == language or language == plan.language))
+      |> Enum.map(fn post ->
+        "<item><title>#{xml_escape(post.title)}</title><link>#{url_for_output(plan.base_url, post_output_path(post))}</link></item>"
+      end)
+      |> Enum.join()
+
+    "<rss><channel><title>#{xml_escape(plan.project_name)} (#{xml_escape(language || "default")})</title>#{items}</channel></rss>"
+  end
+
+  defp render_atom(plan, language, published_posts) do
+    entries =
+      published_posts
+      |> Enum.filter(&(&1.language == language or language == plan.language))
+      |> Enum.map(fn post ->
+        "<entry><title>#{xml_escape(post.title)}</title><id>#{url_for_output(plan.base_url, post_output_path(post))}</id></entry>"
+      end)
+      |> Enum.join()
+
+    "<feed><title>#{xml_escape(plan.project_name)} (#{xml_escape(language || "default")})</title>#{entries}</feed>"
+  end
+
+  defp render_calendar(published_posts) do
+    published_posts
+    |> Enum.map(fn post ->
+      datetime = DateTime.from_unix!(post.created_at)
+      %{date: Date.to_iso8601(DateTime.to_date(datetime)), slug: post.slug, title: post.title}
+    end)
+    |> Jason.encode!()
+  end
+
+  defp render_sitemap(urls) do
+    entries = Enum.map_join(urls, "", fn url -> "<url><loc>#{xml_escape(url)}</loc></url>" end)
+    "<urlset>#{entries}</urlset>"
+  end
+
+  defp render_post_page(title, body, slug, language) do
+    [
+      "<html>",
+      "<head><title>",
+      to_string(title),
+      "</title></head>",
+      "<body data-slug=\"",
+      to_string(slug),
+      "\" data-language=\"",
+      to_string(language),
+      "\"><article data-pagefind-body>",
+      body,
+      "</article></body>",
+      "</html>"
+    ]
+    |> IO.iodata_to_binary()
+  end
+
+  defp load_body(_project_id, _file_path, inline_content) when is_binary(inline_content), do: inline_content
+
+  defp load_body(project_id, file_path, _inline_content) do
+    case file_path do
+      nil -> ""
+      "" -> ""
+      value ->
+        project_path = Path.expand(value, Projects.project_data_dir(Projects.get_project!(project_id)))
+        case File.read(project_path) do
+          {:ok, contents} -> parse_frontmatter_body(contents)
+          {:error, _reason} -> ""
+        end
+    end
+  end
+
+  defp parse_frontmatter_body(contents) do
+    case String.split(contents, "\n---\n", parts: 2) do
+      [_frontmatter, body] -> String.trim_trailing(body, "\n")
+      _parts -> contents
+    end
+  end
+
+  defp url_for_output(nil, relative_path), do: "/" <> String.trim_leading(relative_path, "/")
+
+  defp url_for_output(base_url, relative_path) do
+    cleaned = relative_path |> String.trim_leading("/") |> String.trim_trailing("index.html")
+    suffix = if cleaned == "", do: "/", else: "/" <> cleaned
+    String.trim_trailing(base_url, "/") <> suffix
+  end
+
+  defp xml_escape(value) do
+    value
+    |> to_string()
+    |> String.replace("&", "&amp;")
+    |> String.replace("<", "&lt;")
+    |> String.replace(">", "&gt;")
+    |> String.replace("\"", "&quot;")
+    |> String.replace("'", "&apos;")
+  end
+
   defp output_path(project, relative_path) do
     Path.join([Projects.project_data_dir(project), "html", relative_path])
   end

lib/bds/preview.ex

@@ -0,0 +1,166 @@
+defmodule BDS.Preview do
+  @moduledoc false
+
+  use GenServer
+
+  alias BDS.Posts
+  alias BDS.Projects
+
+  @host "127.0.0.1"
+  @port 4123
+
+  def start_link(_opts) do
+    GenServer.start_link(__MODULE__, %{}, name: __MODULE__)
+  end
+
+  def start_preview(project_id) when is_binary(project_id) do
+    project = Projects.get_project!(project_id)
+    GenServer.call(__MODULE__, {:start_preview, project_id, Projects.project_data_dir(project)})
+  end
+
+  def stop_preview(project_id) when is_binary(project_id) do
+    GenServer.call(__MODULE__, {:stop_preview, project_id})
+  end
+
+  def request(project_id, request_path) when is_binary(project_id) and is_binary(request_path) do
+    GenServer.call(__MODULE__, {:request, project_id, request_path})
+  end
+
+  def preview_draft(project_id, request_path, post_id)
+      when is_binary(project_id) and is_binary(request_path) and is_binary(post_id) do
+    post = Posts.get_post!(post_id)
+    GenServer.call(__MODULE__, {:preview_draft, project_id, request_path, %{title: post.title, body: post.content || ""}})
+  end
+
+  @impl true
+  def init(_state) do
+    {:ok, %{current: nil}}
+  end
+
+  @impl true
+  def handle_call({:start_preview, project_id, data_dir}, _from, state) do
+    server = %{project_id: project_id, data_dir: data_dir, host: @host, port: @port, is_running: true}
+    {:reply, {:ok, server}, %{state | current: server}}
+  end
+
+  def handle_call({:stop_preview, project_id}, _from, state) do
+    next_state =
+      if match?(%{project_id: ^project_id}, state.current) do
+        %{state | current: nil}
+      else
+        state
+      end
+
+    {:reply, :ok, next_state}
+  end
+
+  def handle_call({:request, project_id, request_path}, _from, state) do
+    with :ok <- ensure_running(state.current, project_id),
+         {:ok, response} <- resolve_request(state.current, request_path) do
+      {:reply, {:ok, response}, state}
+    else
+      {:error, reason} -> {:reply, {:error, reason}, state}
+    end
+  end
+
+  def handle_call({:preview_draft, project_id, _request_path, post}, _from, state) do
+    with :ok <- ensure_running(state.current, project_id) do
+      response = %{
+        content_type: "text/html",
+        body: render_draft(post)
+      }
+
+      {:reply, {:ok, response}, state}
+    else
+      {:error, reason} -> {:reply, {:error, reason}, state}
+    end
+  end
+
+  defp ensure_running(%{project_id: project_id, is_running: true}, project_id), do: :ok
+  defp ensure_running(_server, _project_id), do: {:error, :not_running}
+
+  defp resolve_request(server, request_path) do
+    with {:ok, relative_path, kind} <- route_request(request_path) do
+      full_path =
+        case kind do
+          :media -> safe_join(server.data_dir, Path.join(["media", relative_path]))
+          :generated -> safe_join(Path.join(server.data_dir, "html"), relative_path)
+        end
+
+      case full_path do
+        {:error, :not_found} -> {:error, :not_found}
+        resolved_path -> read_response(resolved_path)
+      end
+    end
+  end
+
+  defp route_request(request_path) do
+    normalized = request_path |> URI.parse() |> Map.get(:path, "/")
+    segments = String.split(normalized, "/", trim: true)
+
+    cond do
+      Enum.any?(segments, &(&1 == "..")) ->
+        {:error, :not_found}
+
+      match?(["media" | _], segments) ->
+        {:ok, Path.join(tl(segments)), :media}
+
+      normalized == "/" ->
+        {:ok, "index.html", :generated}
+
+      Path.extname(List.last(segments) || "") != "" ->
+        {:ok, Path.join(segments), :generated}
+
+      true ->
+        {:ok, Path.join(segments ++ ["index.html"]), :generated}
+    end
+  end
+
+  defp safe_join(root, relative_path) do
+    expanded_root = Path.expand(root)
+    expanded_path = Path.expand(relative_path, root)
+
+    if String.starts_with?(expanded_path, expanded_root <> "/") or expanded_path == expanded_root do
+      expanded_path
+    else
+      {:error, :not_found}
+    end
+  end
+
+  defp read_response(path) do
+    case File.read(path) do
+      {:ok, body} -> {:ok, %{body: body, content_type: content_type(path)}}
+      {:error, :enoent} -> {:error, :not_found}
+      {:error, reason} -> {:error, reason}
+    end
+  end
+
+  defp content_type(path) do
+    case Path.extname(path) do
+      ".html" -> "text/html"
+      ".js" -> "application/javascript"
+      ".css" -> "text/css"
+      ".json" -> "application/json"
+      ".xml" -> "application/xml"
+      ".txt" -> "text/plain"
+      ".jpg" -> "image/jpeg"
+      ".jpeg" -> "image/jpeg"
+      ".png" -> "image/png"
+      _other -> "application/octet-stream"
+    end
+  end
+
+  defp render_draft(post) do
+    [
+      "<html>",
+      "<head><title>",
+      to_string(post.title),
+      "</title></head>",
+      "<body><article data-pagefind-body>",
+      post.body,
+      "</article></body>",
+      "</html>"
+    ]
+    |> IO.iodata_to_binary()
+  end
+end

lib/bds/publishing.ex

@@ -0,0 +1,146 @@
+defmodule BDS.Publishing do
+  @moduledoc false
+
+  use GenServer
+
+  alias BDS.Projects
+  alias BDS.Tasks
+
+  def start_link(_opts) do
+    GenServer.start_link(__MODULE__, %{}, name: __MODULE__)
+  end
+
+  def upload_site(project_id, credentials, opts \\ []) when is_binary(project_id) and is_map(credentials) and is_list(opts) do
+    project = Projects.get_project!(project_id)
+    normalized_credentials = normalize_credentials(credentials)
+    targets = build_upload_targets(Projects.project_data_dir(project), normalized_credentials)
+    GenServer.call(__MODULE__, {:upload_site, project_id, normalized_credentials, targets, opts})
+  end
+
+  def get_job(job_id) when is_binary(job_id) do
+    GenServer.call(__MODULE__, {:get_job, job_id})
+  end
+
+  @impl true
+  def init(_state) do
+    {:ok, %{jobs: %{}}}
+  end
+
+  @impl true
+  def handle_call({:get_job, job_id}, _from, state) do
+    {:reply, state.jobs[job_id], state}
+  end
+
+  def handle_call({:update_job, job_id, attrs}, _from, state) do
+    next_state =
+      update_in(state, [:jobs, job_id], fn
+        nil -> nil
+        job -> Map.merge(job, Map.put(attrs, :updated_at, DateTime.utc_now()))
+      end)
+
+    {:reply, :ok, next_state}
+  end
+
+  def handle_call({:upload_site, project_id, credentials, targets, opts}, _from, state) do
+    job_id = "publish-" <> Integer.to_string(System.unique_integer([:positive, :monotonic]))
+    uploader = Keyword.get(opts, :uploader, fn _target, _files, _credentials -> :ok end)
+
+    job = %{
+      id: job_id,
+      project_id: project_id,
+      status: :pending,
+      task_id: nil,
+      ssh_mode: credentials.ssh_mode,
+      targets: Enum.map(targets, & &1.kind),
+      error: nil,
+      inserted_at: DateTime.utc_now(),
+      updated_at: DateTime.utc_now()
+    }
+
+    {:ok, task} =
+      Tasks.submit_task("publish #{project_id}", fn report ->
+        run_upload(job_id, credentials, targets, uploader, report)
+      end, %{
+        group_id: project_id,
+        group_name: "Publishing"
+      })
+
+    next_job = %{job | task_id: task.id}
+    {:reply, {:ok, next_job}, put_in(state, [:jobs, job_id], next_job)}
+  end
+
+  defp run_upload(job_id, credentials, targets, uploader, report) do
+    update_job(job_id, %{status: :running, error: nil})
+
+    result =
+      Enum.with_index(targets, 1)
+      |> Enum.reduce_while(:ok, fn {target, index}, :ok ->
+        files = list_target_files(target)
+        report.(index / max(length(targets), 1), "Uploading #{target.kind}")
+
+        case uploader.(target, files, credentials) do
+          :ok -> {:cont, :ok}
+          {:error, reason} -> {:halt, {:error, reason}}
+        end
+      end)
+
+    case result do
+      :ok ->
+        update_job(job_id, %{status: :completed, error: nil})
+        {:ok, Enum.map(targets, & &1.kind)}
+
+      {:error, reason} ->
+        update_job(job_id, %{status: :failed, error: to_string(reason)})
+        {:error, to_string(reason)}
+    end
+  end
+
+  defp update_job(job_id, attrs) do
+    GenServer.call(__MODULE__, {:update_job, job_id, attrs})
+  end
+
+  defp build_upload_targets(base_dir, credentials) do
+    remote_root = String.trim_trailing(credentials.ssh_remote_path, "/")
+
+    [
+      %{kind: :html, local_dir: Path.join(base_dir, "html"), remote_dir: remote_root},
+      %{kind: :thumbnails, local_dir: Path.join(base_dir, "thumbnails"), remote_dir: Path.join(remote_root, "thumbnails")},
+      %{kind: :media, local_dir: Path.join(base_dir, "media"), remote_dir: Path.join(remote_root, "media")}
+    ]
+  end
+
+  defp list_target_files(target) do
+    if File.dir?(target.local_dir) do
+      target.local_dir
+      |> Path.join("**/*")
+      |> Path.wildcard(match_dot: true)
+      |> Enum.filter(&File.regular?/1)
+      |> Enum.map(&Path.relative_to(&1, target.local_dir))
+      |> Enum.reject(fn relative_path -> target.kind == :media and String.ends_with?(relative_path, ".meta") end)
+      |> Enum.sort()
+    else
+      []
+    end
+  end
+
+  defp normalize_credentials(credentials) do
+    %{
+      ssh_host: attr(credentials, :ssh_host),
+      ssh_user: attr(credentials, :ssh_user),
+      ssh_remote_path: attr(credentials, :ssh_remote_path) || "/",
+      ssh_mode: normalize_ssh_mode(attr(credentials, :ssh_mode))
+    }
+  end
+
+  defp normalize_ssh_mode(mode) when mode in [:scp, :rsync], do: mode
+  defp normalize_ssh_mode("rsync"), do: :rsync
+  defp normalize_ssh_mode(_mode), do: :scp
+
+  defp attr(attrs, key) do
+    cond do
+      Map.has_key?(attrs, key) -> Map.get(attrs, key)
+      Map.has_key?(attrs, Atom.to_string(key)) -> Map.get(attrs, Atom.to_string(key))
+      true -> nil
+    end
+  end
+end