Chunked uploads

bundle/DependencyInjection/Configuration.php

@@ -176,6 +176,16 @@ private function addCloudinaryConfiguration(ArrayNodeDefinition $rootNode): void
                             ->values([CloudinaryProvider::FOLDER_MODE_DYNAMIC, CloudinaryProvider::FOLDER_MODE_FIXED])
                             ->defaultValue(CloudinaryProvider::FOLDER_MODE_DYNAMIC)
                         ->end()
+                        ->integerNode('large_upload_threshold')
+                            ->info('Files strictly larger than this value (in bytes) trigger chunked upload by passing chunk_size to the Cloudinary SDK.')
+                            ->min(1)
+                            ->defaultValue(100_000_000)
+                        ->end()
+                        ->integerNode('upload_chunk_size')
+                            ->info('Chunk size in bytes passed as the chunk_size option to the Cloudinary upload API for files above the threshold.')
+                            ->min(1)
+                            ->defaultValue(20_000_000)
+                        ->end()
                     ->end()
                 ->end()
             ->end();

bundle/DependencyInjection/NetgenRemoteMediaExtension.php

@@ -112,6 +112,16 @@ public function load(array $configs, ContainerBuilder $container): void
             $config['cloudinary']['unique_filenames'],
         );
 
+        $container->setParameter(
+            'netgen_remote_media.cloudinary.large_upload_threshold',
+            $config['cloudinary']['large_upload_threshold'],
+        );
+
+        $container->setParameter(
+            'netgen_remote_media.cloudinary.upload_chunk_size',
+            $config['cloudinary']['upload_chunk_size'],
+        );
+
         if (isset($config['templates'])) {
             if (isset($config['templates']['view_resource'])) {
                 $container->setParameter(

bundle/Resources/config/services/core.yaml

@@ -20,6 +20,8 @@ services:
             - '@netgen_remote_media.provider.cloudinary.factory.search_result'
             - '@netgen_remote_media.provider.cloudinary.resolver.search_expression'
             - '@netgen_remote_media.provider.cloudinary.resolver.auth_token'
+            - '%netgen_remote_media.cloudinary.large_upload_threshold%'
+            - '%netgen_remote_media.cloudinary.upload_chunk_size%'
 
     netgen_remote_media.provider.cloudinary.gateway.cached:
         class: Netgen\RemoteMedia\Core\Provider\Cloudinary\Gateway\Cache\Psr6CachedGateway
@@ -36,6 +38,8 @@ services:
         arguments:
             - "@netgen_remote_media.provider.cloudinary.gateway.api"
             - "@monolog.logger.cloudinary"
+            - '%netgen_remote_media.cloudinary.large_upload_threshold%'
+            - '%netgen_remote_media.cloudinary.upload_chunk_size%'
 
     netgen_remote_media.provider.cloudinary:
         class: Netgen\RemoteMedia\Core\Provider\Cloudinary\CloudinaryProvider

docs/INSTALL.md

@@ -118,6 +118,21 @@ netgen_remote_media:
 
 **WARNING:** if you enable this, all resources will be always unique and you can easily create a mess on your cloud by uploading the same identical file multiple times.
 
+#### Large file uploads
+
+Cloudinary's single-request upload API is capped at 100 MB per file. Files strictly larger than `large_upload_threshold` are uploaded in chunks of `upload_chunk_size` bytes each, by passing the `chunk_size` option to the Cloudinary SDK. Both values are in bytes.
+
+```yaml
+netgen_remote_media:
+    cloudinary:
+        large_upload_threshold: 100000000
+        upload_chunk_size: 20000000
+```
+
+(defaults: `large_upload_threshold: 100000000`, `upload_chunk_size: 20000000`)
+
+**Note:** uploading files larger than 100 MB also requires raising your webserver and PHP body-size limits (eg. nginx `client_max_body_size`, PHP `upload_max_filesize` and `post_max_size`). The bundle itself imposes no such limits. Also check your Cloudinary plan's per-file raw upload cap (40 MB by default); contact Cloudinary support to raise it if needed.
+
 ### Upload prefix
 
 If you need to change Cloudinary API url (to use eg. GEO specific URLs), there's a parameter `upload_prefix` (set to `https://api.cloudinary.com` by default):

lib/Core/Provider/Cloudinary/Gateway/CloudinaryApiGateway.php

@@ -34,8 +34,11 @@
 use function array_merge;
 use function count;
 use function date;
+use function filesize;
 use function floor;
 use function is_array;
+use function is_file;
+use function is_readable;
 use function log;
 use function max;
 use function min;
@@ -55,7 +58,9 @@ public function __construct(
         private RemoteResourceFactoryInterface $remoteResourceFactory,
         private SearchResultFactoryInterface $searchResultFactory,
         private SearchExpressionResolver $searchExpressionResolver,
-        private AuthTokenResolver $authTokenResolver
+        private AuthTokenResolver $authTokenResolver,
+        private int $largeUploadThreshold,
+        private int $uploadChunkSize,
     ) {
         $this->adminApi = new AdminApi();
         $this->uploadApi = new UploadApi();
@@ -172,6 +177,10 @@ public function get(CloudinaryRemoteId $remoteId): RemoteResource
 
     public function upload(string $fileUri, array $options): RemoteResource
     {
+        if ($this->shouldChunk($fileUri)) {
+            $options += ['chunk_size' => $this->uploadChunkSize];
+        }
+
         $response = $this->uploadApi->upload($fileUri, $options);
         $resource = $this->remoteResourceFactory->create((array) $response);
 
@@ -391,6 +400,13 @@ public function getDownloadLink(CloudinaryRemoteId $remoteId, array $options = [
         return (string) Media::fromParams($remoteId->getResourceId(), $options)->toUrl();
     }
 
+    private function shouldChunk(string $fileUri): bool
+    {
+        return is_file($fileUri)
+            && is_readable($fileUri)
+            && filesize($fileUri) > $this->largeUploadThreshold;
+    }
+
     private function formatBytes(int $bytes, int $precision = 2): string
     {
         $units = ['B', 'KB', 'MB', 'GB', 'TB', 'YB'];

lib/Core/Provider/Cloudinary/Gateway/Log/MonologLoggedGateway.php

@@ -14,11 +14,17 @@
 use Psr\Log\LoggerInterface;
 use Psr\Log\NullLogger;
 
+use function filesize;
+use function is_file;
+use function is_readable;
+
 final class MonologLoggedGateway implements GatewayInterface
 {
     public function __construct(
         private GatewayInterface $gateway,
-        private ?LoggerInterface $logger
+        private ?LoggerInterface $logger,
+        private int $largeUploadThreshold,
+        private int $uploadChunkSize,
     ) {
         $this->logger = $this->logger ?? new NullLogger();
     }
@@ -81,7 +87,15 @@ public function get(CloudinaryRemoteId $remoteId): RemoteResource
 
     public function upload(string $fileUri, array $options): RemoteResource
     {
-        $this->logger->info("[API][FREE] upload(\"{$fileUri}\") -> Cloudinary\\Uploader::upload(\"{$fileUri}\")");
+        if (is_file($fileUri) && is_readable($fileUri)) {
+            $size = filesize($fileUri);
+            $chunked = $size > $this->largeUploadThreshold ? 'yes' : 'no';
+            $sizeInfo = "size={$size}B, chunked={$chunked} (threshold={$this->largeUploadThreshold}B, chunk_size={$this->uploadChunkSize}B)";
+        } else {
+            $sizeInfo = 'size=external/unknown, chunked=no';
+        }
+
+        $this->logger->info("[API][FREE] upload(\"{$fileUri}\") [{$sizeInfo}] -> Cloudinary\\Uploader::upload(\"{$fileUri}\")");
 
         return $this->gateway->upload($fileUri, $options);
     }

tests/bundle/DependencyInjection/ConfigurationTest.php

@@ -6,6 +6,7 @@
 
 use Matthias\SymfonyConfigTest\PhpUnit\ConfigurationTestCaseTrait;
 use Netgen\Bundle\RemoteMediaBundle\DependencyInjection\Configuration;
+use Netgen\RemoteMedia\Core\Provider\Cloudinary\CloudinaryProvider;
 use PHPUnit\Framework\Attributes\CoversClass;
 use PHPUnit\Framework\Attributes\DataProvider;
 use PHPUnit\Framework\TestCase;
@@ -152,6 +153,99 @@ public function testMissingAccountKeyIsInvalid(): void
         );
     }
 
+    public function testLargeUploadConfigurationDefaults(): void
+    {
+        $this->assertProcessedConfigurationEquals(
+            [
+                [
+                    'provider' => 'cloudinary',
+                    'account_name' => 'examplename',
+                    'account_key' => 'examplekey',
+                    'account_secret' => 'examplesecret',
+                ],
+            ],
+            [
+                'provider' => 'cloudinary',
+                'account_name' => 'examplename',
+                'account_key' => 'examplekey',
+                'account_secret' => 'examplesecret',
+                'upload_prefix' => 'https://api.cloudinary.com',
+                'remove_unused' => false,
+                'cache' => [
+                    'pool' => 'cache.app',
+                    'ttl' => 7200,
+                ],
+                'cloudinary' => [
+                    'cache_requests' => true,
+                    'log_requests' => false,
+                    'append_extension' => true,
+                    'unique_filenames' => false,
+                    'encryption_key' => null,
+                    'folder_mode' => CloudinaryProvider::FOLDER_MODE_DYNAMIC,
+                    'large_upload_threshold' => 100_000_000,
+                    'upload_chunk_size' => 20_000_000,
+                ],
+                'image_variations' => [],
+            ],
+        );
+    }
+
+    public function testLargeUploadConfigurationOverrides(): void
+    {
+        $this->assertConfigurationIsValid(
+            [
+                'netgen_remote_media' => [
+                    'provider' => 'cloudinary',
+                    'account_name' => 'examplename',
+                    'account_key' => 'examplekey',
+                    'account_secret' => 'examplesecret',
+                    'cloudinary' => [
+                        'large_upload_threshold' => 50_000_000,
+                        'upload_chunk_size' => 10_000_000,
+                    ],
+                ],
+            ],
+        );
+    }
+
+    #[DataProvider('invalidLargeUploadConfigurationProvider')]
+    public function testInvalidLargeUploadConfiguration(array $configuration): void
+    {
+        $this->assertConfigurationIsInvalid($configuration);
+    }
+
+    public static function invalidLargeUploadConfigurationProvider(): iterable
+    {
+        return [
+            'zero threshold' => [
+                [
+                    'netgen_remote_media' => [
+                        'provider' => 'cloudinary',
+                        'account_name' => 'examplename',
+                        'account_key' => 'examplekey',
+                        'account_secret' => 'examplesecret',
+                        'cloudinary' => [
+                            'large_upload_threshold' => 0,
+                        ],
+                    ],
+                ],
+            ],
+            'zero chunk size' => [
+                [
+                    'netgen_remote_media' => [
+                        'provider' => 'cloudinary',
+                        'account_name' => 'examplename',
+                        'account_key' => 'examplekey',
+                        'account_secret' => 'examplesecret',
+                        'cloudinary' => [
+                            'upload_chunk_size' => 0,
+                        ],
+                    ],
+                ],
+            ],
+        ];
+    }
+
     #[DataProvider('invalidNamedObjectsProvider')]
     public function testInvalidNamedObjectsConfiguration(array $configuration): void
     {

tests/bundle/DependencyInjection/NetgenRemoteMediaExtensionTest.php

@@ -27,6 +27,20 @@ public function testItSetsValidContainerParameters(): void
         $this->assertContainerBuilderHasParameter('netgen_remote_media.encryption_key', 'dsf45z45hh45f43f43f');
         $this->assertContainerBuilderHasParameter('netgen_remote_media.cloudinary.append_extension', true);
         $this->assertContainerBuilderHasParameter('netgen_remote_media.cloudinary.unique_filenames', false);
+        $this->assertContainerBuilderHasParameter('netgen_remote_media.cloudinary.large_upload_threshold', 100_000_000);
+        $this->assertContainerBuilderHasParameter('netgen_remote_media.cloudinary.upload_chunk_size', 20_000_000);
+
+        $this->assertContainerBuilderHasServiceDefinitionWithArgument(
+            'netgen_remote_media.provider.cloudinary.gateway.api',
+            5,
+            '%netgen_remote_media.cloudinary.large_upload_threshold%',
+        );
+
+        $this->assertContainerBuilderHasServiceDefinitionWithArgument(
+            'netgen_remote_media.provider.cloudinary.gateway.api',
+            6,
+            '%netgen_remote_media.cloudinary.upload_chunk_size%',
+        );
 
         $this->assertContainerBuilderHasParameter(
             'netgen_remote_media.named_remote_resources',