diff --git a/lib/livebook_web/live/file_select_component.ex b/lib/livebook_web/live/file_select_component.ex
index dcdef0cf223..b8d901be095 100644
--- a/lib/livebook_web/live/file_select_component.ex
+++ b/lib/livebook_web/live/file_select_component.ex
@@ -523,7 +523,7 @@ defmodule LivebookWeb.FileSelectComponent do
     send_event(socket.assigns.target, {:mount_file_system, file_system})
     send_event(socket.assigns.target, {:set_file, file, %{exists: true}})
 
-    {:noreply, assign(socket, loading: true)}
+    {:noreply, socket}
   end
 
   def handle_event("set_path", %{"path" => path}, socket) do
@@ -544,7 +544,7 @@ defmodule LivebookWeb.FileSelectComponent do
       end
 
     send_event(socket.assigns.target, {:set_file, file, info})
-    {:noreply, assign(socket, loading: socket.assigns.file.path != path)}
+    {:noreply, socket}
   end
 
   def handle_event("clear_error", %{}, socket) do
@@ -640,19 +640,31 @@ defmodule LivebookWeb.FileSelectComponent do
     current_file_infos = assigns[:file_infos] || []
     {dir, prefix} = dir_and_prefix(assigns.file)
 
-    {file_infos, socket} =
-      if dir != assigns.current_dir or force_reload? do
-        case get_file_infos(dir, assigns.extnames, assigns.running_files) do
-          {:ok, file_infos} ->
-            {file_infos, assign(socket, :current_dir, dir)}
+    dir_changed? = dir != assigns.current_dir
 
-          {:error, error} ->
-            {current_file_infos, put_error(socket, error)}
-        end
-      else
-        {current_file_infos, socket}
+    if dir_changed? or force_reload? do
+      start_async_file_listing(socket, dir, prefix, current_file_infos)
+    else
+      # Just re-annotate with the current prefix (search filter changed)
+      update_file_display(socket, current_file_infos, prefix)
+    end
+  end
+
+  defp start_async_file_listing(socket, dir, prefix, current_file_infos) do
+    extnames = socket.assigns.extnames
+    running_files = socket.assigns.running_files
+
+    socket
+    |> assign(loading: true)
+    |> start_async(:list_files, fn ->
+      case get_file_infos(dir, extnames, running_files) do
+        {:ok, file_infos} -> {:ok, file_infos, dir, prefix}
+        {:error, error} -> {:error, error, current_file_infos, prefix}
       end
+    end)
+  end
 
+  defp update_file_display(socket, file_infos, prefix) do
     file_infos = annotate_matching(file_infos, prefix)
 
     {unhighlighted_file_infos, highlighted_file_infos} =
@@ -665,11 +677,40 @@ defmodule LivebookWeb.FileSelectComponent do
     assign(socket,
       file_infos: file_infos,
       unhighlighted_file_infos: unhighlighted_file_infos,
-      highlighted_file_infos: highlighted_file_infos,
-      loading: false
+      highlighted_file_infos: highlighted_file_infos
     )
   end
 
+  @impl true
+  def handle_async(:list_files, {:ok, {:ok, file_infos, dir, prefix}}, socket) do
+    socket =
+      socket
+      |> assign(current_dir: dir)
+      |> update_file_display(file_infos, prefix)
+      |> assign(loading: false)
+
+    {:noreply, socket}
+  end
+
+  def handle_async(:list_files, {:ok, {:error, error, current_file_infos, prefix}}, socket) do
+    socket =
+      socket
+      |> update_file_display(current_file_infos, prefix)
+      |> put_error(error)
+      |> assign(loading: false)
+
+    {:noreply, socket}
+  end
+
+  def handle_async(:list_files, {:exit, _reason}, socket) do
+    socket =
+      socket
+      |> put_error("Listing files failed")
+      |> assign(loading: false)
+
+    {:noreply, socket}
+  end
+
   defp annotate_matching(file_infos, prefix) do
     for %{name: name} = info <- file_infos do
       case String.split(name, prefix, parts: 2) do
diff --git a/test/livebook_web/live/file_select_component_test.exs b/test/livebook_web/live/file_select_component_test.exs
index 946e38330e9..bef899fad39 100644
--- a/test/livebook_web/live/file_select_component_test.exs
+++ b/test/livebook_web/live/file_select_component_test.exs
@@ -5,37 +5,69 @@ defmodule LivebookWeb.FileSelectComponentTest do
   import Livebook.TestHelpers
 
   alias Livebook.FileSystem
-  alias LivebookWeb.FileSelectComponent
 
-  test "when the path has a trailing slash, lists that directory" do
+  test "when the path has a trailing slash, lists that directory", %{conn: conn} do
     file = FileSystem.File.local(notebooks_path() <> "/")
-    assert render_component(FileSelectComponent, attrs(file: file)) =~ "basic.livemd"
-    assert render_component(FileSelectComponent, attrs(file: file)) =~ ".."
+    html = render_file_select(conn, file)
+
+    assert html =~ "basic.livemd"
+    assert html =~ ".."
   end
 
-  test "when the path has no trailing slash, lists the parent directory" do
+  test "when the path has no trailing slash, lists the parent directory", %{conn: conn} do
     file = FileSystem.File.local(notebooks_path())
-    assert render_component(FileSelectComponent, attrs(file: file)) =~ "notebooks"
+
+    assert render_file_select(conn, file) =~ "notebooks"
   end
 
-  test "does not show parent directory when in root" do
+  test "does not show parent directory when in root", %{conn: conn} do
     file = FileSystem.File.local(p("/"))
-    refute render_component(FileSelectComponent, attrs(file: file)) =~ ".."
+
+    refute render_file_select(conn, file) =~ ".."
   end
 
-  defp attrs(attrs) do
-    Keyword.merge(
-      [
-        id: "1",
-        file: FileSystem.File.local(p("/")),
-        extnames: [".livemd"],
-        running_files: []
-      ],
-      attrs
-    )
+  defp render_file_select(conn, file) do
+    {:ok, view, _html} =
+      live_isolated(conn, LivebookWeb.FileSelectComponentTest.Live,
+        session: %{"path" => file.path}
+      )
+
+    render_async(view)
   end
 
   defp notebooks_path() do
     Path.expand("../../support/notebooks", __DIR__)
   end
+
+  defmodule Live do
+    use Phoenix.LiveView, layout: false
+
+    alias Livebook.FileSystem
+    alias LivebookWeb.FileSelectComponent
+
+    @impl true
+    def mount(_params, %{"path" => path}, socket) do
+      socket =
+        assign(socket,
+          file: FileSystem.File.local(path),
+          extnames: [".livemd"],
+          running_files: []
+        )
+
+      {:ok, socket}
+    end
+
+    @impl true
+    def render(assigns) do
+      ~H"""
+      <.live_component
+        module={FileSelectComponent}
+        id="1"
+        file={@file}
+        extnames={@extnames}
+        running_files={@running_files}
+      />
+      """
+    end
+  end
 end