fix: issue #10 incomplete upload

lib/bds/publishing.ex

@@ -181,7 +181,7 @@ defmodule BDS.Publishing do
          ssh_auth_sock
        ) do
     args =
-      ["--update", "--compress", "--verbose"] ++
+      ["--update", "--compress", "--verbose", "--recursive", "--times"] ++
         rsync_excludes(target) ++
         [
           "-e",
@@ -198,25 +198,39 @@ defmodule BDS.Publishing do
       files_to_upload =
         filter_scp_uploads(project_id, credentials, target.kind, files_with_mtimes)
 
-      case upload_scp_files(
-             project_id,
-             target,
-             credentials,
-             runner,
-             ssh_auth_sock,
-             files_to_upload,
-             []
-           ) do
-        {:ok, uploaded_files} ->
-          persist_uploaded_scp_files(project_id, credentials, target.kind, uploaded_files)
-          :ok
-
-        {:error, reason} ->
-          {:error, reason}
+      with :ok <-
+             ensure_remote_dirs(target, credentials, runner, ssh_auth_sock, files_to_upload) do
+        do_upload_scp_files(
+          project_id,
+          target,
+          credentials,
+          runner,
+          ssh_auth_sock,
+          files_to_upload
+        )
       end
     end
   end
 
+  defp do_upload_scp_files(project_id, target, credentials, runner, ssh_auth_sock, files_to_upload) do
+    case upload_scp_files(
+           project_id,
+           target,
+           credentials,
+           runner,
+           ssh_auth_sock,
+           files_to_upload,
+           []
+         ) do
+      {:ok, uploaded_files} ->
+        persist_uploaded_scp_files(project_id, credentials, target.kind, uploaded_files)
+        :ok
+
+      {:error, reason} ->
+        {:error, reason}
+    end
+  end
+
   defp run_command(runner, command, args, ssh_auth_sock) do
     opts = command_opts(ssh_auth_sock)
     {output, exit_status} = runner.(command, args, opts)
@@ -302,6 +316,25 @@ defmodule BDS.Publishing do
     end
   end
 
+  # scp does not create missing remote directories, so create them up front.
+  defp ensure_remote_dirs(_target, _credentials, _runner, _ssh_auth_sock, []), do: :ok
+
+  defp ensure_remote_dirs(target, credentials, runner, ssh_auth_sock, files_to_upload) do
+    remote_dirs =
+      files_to_upload
+      |> Enum.map(fn {relative_path, _local_mtime} ->
+        target.remote_dir |> Path.join(relative_path) |> Path.dirname()
+      end)
+      |> Enum.uniq()
+
+    run_command(
+      runner,
+      "ssh",
+      [remote_base(credentials), "mkdir", "-p" | remote_dirs],
+      ssh_auth_sock
+    )
+  end
+
   defp collect_file_mtimes(local_dir, files) do
     Enum.reduce_while(files, {:ok, []}, fn relative_path, {:ok, acc} ->
       local_path = Path.join(local_dir, relative_path)

test/bds/publishing_test.exs

@@ -99,6 +99,8 @@ defmodule BDS.PublishingTest do
              "--update",
              "--compress",
              "--verbose",
+             "--recursive",
+             "--times",
              "-e",
              "ssh",
              Path.join([temp_dir, "html"]) <> "/",
@@ -113,6 +115,8 @@ defmodule BDS.PublishingTest do
              "--update",
              "--compress",
              "--verbose",
+             "--recursive",
+             "--times",
              "-e",
              "ssh",
              Path.join([temp_dir, "thumbnails"]) <> "/",
@@ -125,6 +129,8 @@ defmodule BDS.PublishingTest do
              "--update",
              "--compress",
              "--verbose",
+             "--recursive",
+             "--times",
              "--exclude=*.meta",
              "-e",
              "ssh",
@@ -175,6 +181,10 @@ defmodule BDS.PublishingTest do
     failed_job = wait_for_publish_job(job.id, &(&1.status == :failed))
     assert failed_job.error =~ "thumbnail failure"
 
+    assert_receive {:command_run, "ssh",
+                    ["deploy@example.com", "mkdir", "-p", "/srv/blog", "/srv/blog/posts"],
+                    _mkdir_opts}
+
     assert_receive {:command_run, "scp",
                     ["-q", ^html_index, "deploy@example.com:/srv/blog/index.html"], opts_a}
 
@@ -254,7 +264,7 @@ defmodule BDS.PublishingTest do
 
     assert wait_for_publish_job(first_job.id, &(&1.status == :completed)).status == :completed
     first_uploads = collect_command_runs()
-    assert length(first_uploads) == 3
+    assert Enum.count(first_uploads, fn {command, _args} -> command == "scp" end) == 3
 
     assert {:ok, second_job} =
              BDS.Publishing.upload_site(project.id, credentials,
@@ -275,8 +285,9 @@ defmodule BDS.PublishingTest do
 
     assert wait_for_publish_job(third_job.id, &(&1.status == :completed)).status == :completed
 
-    assert [html_upload] = collect_command_runs()
-    assert elem(html_upload, 0) == "scp"
+    third_uploads = collect_command_runs()
+
+    assert [html_upload] = Enum.filter(third_uploads, fn {command, _args} -> command == "scp" end)
     assert elem(html_upload, 1) == ["-q", html_index, "deploy@example.com:/srv/blog/index.html"]
   end