feat: pipeline file parts within each download connection (#107)

* feat: pipeline file parts within each download connection

Workers requested their block's 1MB parts sequentially, paying a full
round-trip of dead time per part and capping each connection around
3-4 MB/s regardless of its server-side allowance - the multi-connection
download barely improved on a single connection (~10 MB/s with dips).

Request all parts of a block concurrently on the block's connection,
keeping each connection's pipe full (the same pipelining official
clients use), and log a completion summary with the effective average
speed to make future tuning measurable. Part writes stay positional
and block completion still gates the contiguous confirmed prefix used
for progress and resume persistence.

* feat: make multi-connection tuning configurable with documented defaults

Expose the remaining hardcoded transfer knobs in General Config: chunk
size (snapped to Telegram's allowed 128/256/512/1024 KB values, 512
being WTelegramClient's own default), block size per connection (which
determines the requests in flight per connection) and the minimum file
size for multi-connection mode. Values are captured once per download
so a config change cannot desynchronize offsets mid-transfer.

The Config page now states the library default, app default and
recommended value for each transfer setting, so the stock WTelegramClient
behavior (2 parallel chunks, 512KB parts, single connection) can be
restored by configuration alone.

Author: mateof

TelegramDownloader/Data/TelegramService.cs

@@ -271,19 +271,42 @@ private class DownloadPool
         }
 
         private static readonly DownloadPool downloadPool = new DownloadPool();
-        private const int MULTICONN_PART_SIZE = 1024 * 1024;        // upload.getFile max limit per request
-        private const int MULTICONN_BLOCK_SIZE = 4 * 1024 * 1024;   // work unit assigned to a connection
-        private const long MULTICONN_MIN_FILE_SIZE = 32L * 1024 * 1024;
 
         public static int GetConfiguredDownloadConnections()
         {
             return Math.Clamp(GeneralConfigStatic.config?.DownloadConnections ?? 4, 2, 8);
         }
 
+        /// <summary>
+        /// Chunk size for upload.getFile: Telegram only accepts limits that are
+        /// divisible by 4KB and divide 1MB evenly, so the configured value is
+        /// snapped to 128/256/512/1024 KB. 512 is WTelegramClient's own default;
+        /// 1024 (app default) halves the number of round-trips.
+        /// </summary>
+        private static int GetConfiguredPartSize()
+        {
+            int kb = GeneralConfigStatic.config?.MultiConnectionPartSizeKB ?? 1024;
+            if (kb >= 1024) return 1024 * 1024;
+            if (kb >= 512) return 512 * 1024;
+            if (kb >= 256) return 256 * 1024;
+            return 128 * 1024;
+        }
+
+        private static int GetConfiguredBlockSize(int partSize)
+        {
+            int mb = Math.Clamp(GeneralConfigStatic.config?.MultiConnectionBlockSizeMB ?? 4, 1, 16);
+            return Math.Max(mb * 1024 * 1024, partSize);
+        }
+
+        private static long GetConfiguredMinFileSize()
+        {
+            return Math.Max(1, GeneralConfigStatic.config?.MultiConnectionMinFileSizeMB ?? 32) * 1024L * 1024L;
+        }
+
         private static bool ShouldUseMultiConnection(TL.Document document)
         {
             GeneralConfig cfg = GeneralConfigStatic.config;
-            return cfg != null && cfg.EnableMultiConnectionDownloads && document.size >= MULTICONN_MIN_FILE_SIZE;
+            return cfg != null && cfg.EnableMultiConnectionDownloads && document.size >= GetConfiguredMinFileSize();
         }
 
         private async Task<List<WTelegram.Client>> GetDownloadPoolAsync(int count)
@@ -448,33 +471,68 @@ private async Task<bool> TryMultiConnectionDownloadAsync(TL.Document document, F
                 thumb_size = ""
             };
 
-            _logger.LogInformation("Multi-connection download - FileName: {Name}, Size: {SizeMB:F2}MB, Connections: {Connections}",
-                model.name, size / (1024.0 * 1024.0), transfers.Count);
+            // Capture the tuning values once so a config change mid-download
+            // cannot desynchronize offsets.
+            int partSize = GetConfiguredPartSize();
+            int blockSize = GetConfiguredBlockSize(partSize);
 
-            long blockCount = (size + MULTICONN_BLOCK_SIZE - 1) / MULTICONN_BLOCK_SIZE;
+            _logger.LogInformation("Multi-connection download - FileName: {Name}, Size: {SizeMB:F2}MB, Connections: {Connections}, Part: {PartKB}KB, Block: {BlockMB}MB",
+                model.name, size / (1024.0 * 1024.0), transfers.Count, partSize / 1024, blockSize / (1024.0 * 1024.0));
+
+            long blockCount = (size + blockSize - 1) / blockSize;
             bool[] blockDone = new bool[blockCount];
             long confirmedBlocks = 0;
             long nextBlock = -1;
             object progressLock = new object();
             using CancellationTokenSource cts = new CancellationTokenSource();
             dest.SetLength(size);
             var handle = dest.SafeFileHandle;
+            DateTime started = DateTime.Now;
 
-            void ReportPart(long block, int received, bool blockCompleted)
+            void ReportBytes(int received)
+            {
+                long confirmed;
+                lock (progressLock)
+                    confirmed = Math.Min(size, confirmedBlocks * (long)blockSize);
+                // Throws when the task gets canceled or paused, stopping the workers.
+                model.ReportParallelProgress(confirmed, received, size);
+            }
+
+            void ReportBlockDone(long block)
             {
                 long confirmed;
                 lock (progressLock)
                 {
-                    if (blockCompleted)
+                    blockDone[block] = true;
+                    while (confirmedBlocks < blockCount && blockDone[confirmedBlocks])
+                        confirmedBlocks++;
+                    confirmed = Math.Min(size, confirmedBlocks * (long)blockSize);
+                }
+                model.ReportParallelProgress(confirmed, 0, size);
+            }
+
+            async Task DownloadPart(WTelegram.Client pc, long offset)
+            {
+                int expected = (int)Math.Min(partSize, size - offset);
+                for (int attempt = 1; ; attempt++)
+                {
+                    cts.Token.ThrowIfCancellationRequested();
+                    try
                     {
-                        blockDone[block] = true;
-                        while (confirmedBlocks < blockCount && blockDone[confirmedBlocks])
-                            confirmedBlocks++;
+                        Upload_FileBase resp = await pc.Upload_GetFile(location, offset, limit: partSize);
+                        if (resp is not Upload_File part)
+                            throw new InvalidOperationException($"Unexpected {resp?.GetType().Name} from Upload_GetFile (CDN-served files are not supported)");
+                        if (part.bytes.Length < expected)
+                            throw new InvalidOperationException($"Short chunk at offset {offset}: {part.bytes.Length} < {expected}");
+                        RandomAccess.Write(handle, part.bytes.AsSpan(0, expected), offset);
+                        ReportBytes(expected);
+                        return;
+                    }
+                    catch (Exception) when (attempt < 3 && !cts.IsCancellationRequested)
+                    {
+                        await Task.Delay(1000 * attempt);
                     }
-                    confirmed = Math.Min(size, confirmedBlocks * (long)MULTICONN_BLOCK_SIZE);
                 }
-                // Throws when the task gets canceled or paused, stopping the workers.
-                model.ReportParallelProgress(confirmed, received, size);
             }
 
             async Task Worker(WTelegram.Client pc)
@@ -484,32 +542,17 @@ async Task Worker(WTelegram.Client pc)
                     long block = Interlocked.Increment(ref nextBlock);
                     if (block >= blockCount)
                         return;
-                    long offset = block * (long)MULTICONN_BLOCK_SIZE;
-                    long end = Math.Min(size, offset + MULTICONN_BLOCK_SIZE);
-                    while (offset < end)
-                    {
-                        cts.Token.ThrowIfCancellationRequested();
-                        Upload_FileBase resp = null;
-                        for (int attempt = 1; ; attempt++)
-                        {
-                            try
-                            {
-                                resp = await pc.Upload_GetFile(location, offset, limit: MULTICONN_PART_SIZE);
-                                break;
-                            }
-                            catch (Exception) when (attempt < 3 && !cts.IsCancellationRequested)
-                            {
-                                await Task.Delay(1000 * attempt);
-                            }
-                        }
-                        if (resp is not Upload_File part)
-                            throw new InvalidOperationException($"Unexpected {resp?.GetType().Name} from Upload_GetFile (CDN-served files are not supported)");
-                        if (part.bytes.Length == 0)
-                            throw new InvalidOperationException($"Empty chunk at offset {offset}");
-                        RandomAccess.Write(handle, part.bytes, offset);
-                        offset += part.bytes.Length;
-                        ReportPart(block, part.bytes.Length, offset >= end);
-                    }
+                    long blockStart = block * (long)blockSize;
+                    long blockEnd = Math.Min(size, blockStart + blockSize);
+                    // Request every part of the block concurrently on this
+                    // connection: sequential parts pay a full round-trip of dead
+                    // time each, capping a connection well below its server-side
+                    // allowance. Pipelining keeps the connection's pipe full.
+                    List<Task> parts = new List<Task>();
+                    for (long offset = blockStart; offset < blockEnd; offset += partSize)
+                        parts.Add(DownloadPart(pc, offset));
+                    await Task.WhenAll(parts);
+                    ReportBlockDone(block);
                 }
             }
 
@@ -523,6 +566,9 @@ async Task GuardedWorker(WTelegram.Client pc)
             {
                 await Task.WhenAll(transfers.Select(pc => Task.Run(() => GuardedWorker(pc))));
                 await dest.FlushAsync();
+                double seconds = Math.Max(0.001, (DateTime.Now - started).TotalSeconds);
+                _logger.LogInformation("Multi-connection download completed - FileName: {Name}, {SizeMB:F1}MB in {Seconds:F1}s = {Speed:F1} MB/s over {Connections} connections",
+                    model.name, size / (1024.0 * 1024.0), seconds, size / (1024.0 * 1024.0) / seconds, transfers.Count);
                 return true;
             }
             catch (Exception ex)

TelegramDownloader/Models/GeneralConfig.cs

@@ -204,9 +204,36 @@ public StreamingMode GetEffectiveStreamingMode()
         /// <summary>
         /// Number of parallel connections used per file download (2-8).
         /// Only used when EnableMultiConnectionDownloads is true.
+        /// App default: 4. Recommended: 4-8.
         /// </summary>
         public int DownloadConnections { get; set; } = 4;
 
+        /// <summary>
+        /// Size in KB of each file chunk requested (upload.getFile limit).
+        /// Telegram only allows 128, 256, 512 or 1024 (values are snapped to
+        /// the nearest allowed one). WTelegramClient's own default is 512;
+        /// app default is 1024 (fewer round-trips).
+        /// Only used when EnableMultiConnectionDownloads is true.
+        /// </summary>
+        public int MultiConnectionPartSizeKB { get; set; } = 1024;
+
+        /// <summary>
+        /// Size in MB (1-16) of the work unit assigned to each connection.
+        /// All parts of a block are requested in parallel on the same
+        /// connection, so BlockSize / PartSize = requests in flight per
+        /// connection. App default: 4 (= 4 x 1MB in flight).
+        /// Only used when EnableMultiConnectionDownloads is true.
+        /// </summary>
+        public int MultiConnectionBlockSizeMB { get; set; } = 4;
+
+        /// <summary>
+        /// Files smaller than this size in MB use the normal single-connection
+        /// download (the setup cost is not worth it for small files).
+        /// App default: 32.
+        /// Only used when EnableMultiConnectionDownloads is true.
+        /// </summary>
+        public int MultiConnectionMinFileSizeMB { get; set; } = 32;
+
     }
 
     public class TLConfig

TelegramDownloader/Pages/Config.razor

@@ -237,7 +237,8 @@
                                 Parallel Chunk Transfers
                             </div>
                             <div class="config-item-description">
-                                Number of 512KB chunks requested in parallel per transfer (1-16). Higher values improve download/upload speed by removing the latency bottleneck; Premium accounts benefit the most. Applied on the next transfer.
+                                Number of chunks requested in parallel per transfer (1-16), used by standard (single-connection) downloads and uploads. Higher values remove the latency bottleneck; Premium accounts benefit the most. Applied on the next transfer.
+                                <br /><small class="text-muted">WTelegramClient default: <b>2</b> &middot; App default: <b>4</b> &middot; Recommended: <b>4-8</b></small>
                             </div>
                         </div>
                         <div class="config-item-control">
@@ -252,10 +253,11 @@
                                 Multi-Connection Downloads
                             </div>
                             <div class="config-item-description">
-                                Download large files (&gt;32MB) using several parallel connections, like Telegram Desktop does.
+                                Download large files using several parallel connections, like Telegram Desktop does.
                                 Telegram limits speed per connection (~5-6 MB/s), so this is the way to go faster.
                                 The extra connections share your existing session authorization, so nothing new appears in your Telegram device list.
                                 <span class="badge bg-warning text-dark ms-2">Experimental</span>
+                                <br /><small class="text-muted">Default: <b>disabled</b> (library behavior: one connection per download) &middot; Recommended: <b>enabled</b> for large files on fast lines</small>
                             </div>
                         </div>
                         <div class="config-item-control">
@@ -272,13 +274,73 @@
                                     Download Connections
                                 </div>
                                 <div class="config-item-description">
-                                    Number of parallel connections used per file download (2-8)
+                                    Number of parallel connections used per file download (2-8).
+                                    <br /><small class="text-muted">App default: <b>4</b> &middot; Recommended: <b>4-8</b> (Telegram Desktop uses up to 8)</small>
                                 </div>
                             </div>
                             <div class="config-item-control">
                                 <NumberInput TValue="int" @bind-Value="Model!.DownloadConnections" Min="2" Max="8" EnableMinMax="true" class="form-control" style="width: 80px;" />
                             </div>
                         </div>
+
+                        <div class="config-item">
+                            <div class="config-item-info">
+                                <div class="config-item-label">
+                                    <i class="bi bi-hdd-stack"></i>
+                                    Chunk Size
+                                </div>
+                                <div class="config-item-description">
+                                    Size of each file chunk requested from Telegram. Bigger chunks mean fewer round-trips.
+                                    <br /><small class="text-muted">WTelegramClient default: <b>512 KB</b> &middot; App default and recommended: <b>1024 KB</b></small>
+                                </div>
+                            </div>
+                            <div class="config-item-control">
+                                <InputSelect @bind-Value="Model!.MultiConnectionPartSizeKB" class="form-select" style="width: 120px;">
+                                    <option value="128">128 KB</option>
+                                    <option value="256">256 KB</option>
+                                    <option value="512">512 KB</option>
+                                    <option value="1024">1024 KB</option>
+                                </InputSelect>
+                            </div>
+                        </div>
+
+                        <div class="config-item">
+                            <div class="config-item-info">
+                                <div class="config-item-label">
+                                    <i class="bi bi-collection"></i>
+                                    Block Size
+                                </div>
+                                <div class="config-item-description">
+                                    Work unit assigned to each connection (1-16 MB). All chunks of a block are requested in parallel on the same connection, so Block &divide; Chunk = requests in flight per connection.
+                                    <br /><small class="text-muted">App default: <b>4 MB</b> (= 4 chunks of 1024 KB in flight) &middot; Recommended: <b>4-8 MB</b></small>
+                                </div>
+                            </div>
+                            <div class="config-item-control">
+                                <div class="config-input-group">
+                                    <NumberInput TValue="int" @bind-Value="Model!.MultiConnectionBlockSizeMB" Min="1" Max="16" EnableMinMax="true" class="form-control" style="width: 80px;" />
+                                    <span class="text-muted">MB</span>
+                                </div>
+                            </div>
+                        </div>
+
+                        <div class="config-item">
+                            <div class="config-item-info">
+                                <div class="config-item-label">
+                                    <i class="bi bi-rulers"></i>
+                                    Min. File Size
+                                </div>
+                                <div class="config-item-description">
+                                    Files smaller than this use the normal single-connection download (the setup is not worth it for small files).
+                                    <br /><small class="text-muted">App default and recommended: <b>32 MB</b></small>
+                                </div>
+                            </div>
+                            <div class="config-item-control">
+                                <div class="config-input-group">
+                                    <NumberInput TValue="int" @bind-Value="Model!.MultiConnectionMinFileSizeMB" Min="1" Max="1024" EnableMinMax="true" class="form-control" style="width: 90px;" />
+                                    <span class="text-muted">MB</span>
+                                </div>
+                            </div>
+                        </div>
                     }
 
                     <div class="config-item">