Validate Rclone remotes in initializer (#2952)

Validates by trying to query in parallel with a timeout of 6s. Disabled
by default, enable by setting OOD_REMOTE_FILES_VALIDATION.

Author: robinkar

apps/dashboard/config/configuration_singleton.rb

@@ -44,6 +44,7 @@ def boolean_configs
       :bc_clean_old_dirs            => false,
       :per_cluster_dataroot         => false,
       :remote_files_enabled         => false,
+      :remote_files_validation      => false,
       :host_based_profiles          => false,
       :disable_bc_shell             => false,
       :cancel_session_enabled       => false,

apps/dashboard/config/initializers/rclone.rb

@@ -3,10 +3,21 @@
 Rails.application.config.after_initialize do
   next unless Configuration.remote_files_enabled?
 
-  remotes = RcloneUtil.list_remotes.map { |r| FavoritePath.new('', title: r, filesystem: r) }
+  remotes = RcloneUtil.list_remotes
+
+  if Configuration.remote_files_validation?
+    # Query remotes in parallel
+    mutex = Mutex.new
+    remotes.map do |remote|
+      Thread.new do
+        valid = RcloneUtil.valid?(remote)
+        mutex.synchronize { remotes.delete(remote) } unless valid
+      end
+    end.each(&:join)
+  end
 
   OodFilesApp.candidate_favorite_paths.tap do |paths|
-    paths.concat(remotes)
+    paths.concat(remotes.map { |r| FavoritePath.new('', title: r, filesystem: r) })
   end
 rescue => e
   Rails.logger.error("Cannot add rclone favorite paths because #{e.message}")

apps/dashboard/lib/rclone_util.rb

@@ -237,6 +237,21 @@ def list_remotes
       end
     end
 
+    # Check if remote can be accessed (responds to directory listing request).
+    def valid?(remote)
+      # This gives a total max duration of 6s (2s * 3 attempts)
+      _, _, s = rclone(
+        'lsd',                 "#{remote}:",
+        '--contimeout',        '2s',
+        '--timeout',           '2s',
+        '--low-level-retries', '1',
+        '--retries',           '1'
+      )
+      s.success?
+    rescue StandardError
+      false
+    end
+
     def rclone_cmd
       # TODO: Make this configurable
       "rclone"

apps/dashboard/test/fixtures/config/ondemand.d/booleans.yml

@@ -5,6 +5,7 @@ bc_simple_auto_accounts: true
 per_cluster_dataroot: true
 file_navigator: true
 remote_files_enabled: true
+remote_files_validation: true
 host_based_profiles: true
 disable_bc_shell: true
 cancel_session_enabled: true

apps/dashboard/test/fixtures/config/rclone/rclone.conf

@@ -5,3 +5,8 @@ env_auth = false
 access_key_id = 
 secret_access_key = 
 acl = private
+
+# S3 always tried to query endpoint, use swift for testing invalid remotes.
+[missing_auth]
+type = swift
+env_auth = false

apps/dashboard/test/unit/rclone_util_test.rb

@@ -5,7 +5,7 @@
 class RcloneUtilTest < ActiveSupport::TestCase
   test "list_remotes handles rclone.conf and env" do
     with_rclone_conf("/") do
-        assert_equal ["alias_remote", "local_remote", "s3"], RcloneUtil.list_remotes
+        assert_equal ["alias_remote", "local_remote", "missing_auth", "s3"], RcloneUtil.list_remotes
     end
   end
 
@@ -14,4 +14,12 @@ class RcloneUtilTest < ActiveSupport::TestCase
         assert_equal [], RcloneUtil.list_remotes
     end
   end
+
+  test 'missing remote or missing auth is considered invalid' do
+    with_rclone_conf('/') do
+      refute RcloneUtil.valid?('missing_remote')
+      refute RcloneUtil.valid?('missing_auth')
+      assert RcloneUtil.valid?('local_remote')
+    end
+  end
 end