Increase the number of threads to 10 (default for S3) (#1423)

Author: dimaryaz

api/python/quilt3/data_transfer.py

@@ -48,7 +48,6 @@ def create_s3_client():
 
 
 s3_transfer_config = TransferConfig()
-s3_threads = 4
 
 # When uploading files at least this size, compare the ETags first and skip the upload if they're equal;
 # copy the remote file onto itself if the metadata changes.
@@ -277,7 +276,7 @@ def _copy_file_list_internal(s3_client, file_list):
     results = [None] * len(file_list)
 
     with tqdm(desc="Copying", total=total_size, unit='B', unit_scale=True) as progress, \
-         ThreadPoolExecutor(s3_threads) as executor:
+         ThreadPoolExecutor(s3_transfer_config.max_request_concurrency) as executor:
 
         def progress_callback(bytes_transferred):
             with lock:

api/python/tests/integration/test_packages.py

@@ -380,7 +380,7 @@ def test_load_into_quilt(self):
         new_pkg.set('foo2', test_file1)
 
         with patch('time.time', return_value=1234567890), \
-             patch('quilt3.data_transfer.s3_threads', 1):
+             patch('quilt3.data_transfer.s3_transfer_config.max_request_concurrency', 1):
             remote_pkg = new_pkg.push('Quilt/package', 's3://my_test_bucket/')
 
         # Modify one file, and check that only that file gets uploaded.
@@ -439,7 +439,7 @@ def test_load_into_quilt(self):
         remote_pkg.set('foo2', test_file3)
 
         with patch('time.time', return_value=1234567891), \
-             patch('quilt3.data_transfer.s3_threads', 1):
+             patch('quilt3.data_transfer.s3_transfer_config.max_request_concurrency', 1):
             remote_pkg.push('Quilt/package', 's3://my_test_bucket/')
 
 

api/python/tests/test_data_transfer.py

@@ -199,7 +199,7 @@ def test_multi_upload(self):
         )
 
         # stubber expects responses in order, so disable multi-threading.
-        with mock.patch('quilt3.data_transfer.s3_threads', 1):
+        with mock.patch('quilt3.data_transfer.s3_transfer_config.max_request_concurrency', 1):
             urls = data_transfer.copy_file_list([
                 (path1.as_uri(), 's3://example1/foo.csv', path1.stat().st_size),
                 (path2.as_uri(), 's3://example2/foo.txt', path2.stat().st_size),
@@ -360,7 +360,7 @@ def test_multipart_upload(self):
             }
         )
 
-        with mock.patch('quilt3.data_transfer.s3_threads', 1):
+        with mock.patch('quilt3.data_transfer.s3_transfer_config.max_request_concurrency', 1):
             data_transfer.copy_file_list([
                 (path.resolve().as_uri(), f's3://example/{name}', path.stat().st_size),
             ])
@@ -429,7 +429,7 @@ def test_multipart_copy(self):
             }
         )
 
-        with mock.patch('quilt3.data_transfer.s3_threads', 1):
+        with mock.patch('quilt3.data_transfer.s3_transfer_config.max_request_concurrency', 1):
             data_transfer.copy_file_list([
                 ('s3://example1/large_file1.npy', 's3://example2/large_file2.npy', size),
             ])