1. 项目概述：基于asyncio的并发下载器
在Python生态中，文件下载是常见的IO密集型任务场景。传统同步下载方式在面对批量下载需求时，会因网络延迟和磁盘IO等待导致严重的性能瓶颈。我曾在一个爬虫项目中需要下载10,000+个PDF文件，使用requests同步下载耗时超过6小时，而改用asyncio后仅需18分钟——这正是异步编程的魅力所在。
asyncio作为Python标准库中的异步I/O框架，其核心优势在于：
- 单线程内通过事件循环调度多个协程
- 遇到IO阻塞时自动切换任务上下文
- 用同步代码写法实现异步执行效果
本示例将构建一个支持并发控制的下载器，主要解决三个实际问题：
- 如何避免同时发起过多连接导致服务器拒绝服务
- 如何处理下载中断后的断点续传
- 如何实现进度监控与速度限制
2. 核心架构设计
2.1 异步HTTP客户端选型
对比常见异步HTTP库的基准测试数据（测试环境：Python 3.10，100个1MB文件下载）：
| 库名称 | 平均耗时(s) | 内存峰值(MB) | 特点 |
|---|---|---|---|
| aiohttp | 12.3 | 45 | 功能完整，社区活跃 |
| httpx | 14.7 | 52 | 同步/异步统一API |
| requests+线程池 | 28.5 | 110 | 同步方案对比组 |
选择aiohttp的核心考量：
python复制async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
# 内置连接池管理
# 自动处理chunked传输
# 支持SSL配置
2.2 并发控制实现
采用信号量(Semaphore)控制最大并发数：
python复制class Downloader:
def __init__(self, max_concurrent=10):
self.semaphore = asyncio.Semaphore(max_concurrent)
async def download(self, url):
async with self.semaphore: # 关键控制点
async with aiohttp.ClientSession() as session:
return await self._fetch(session, url)
2.3 断点续传机制
通过HTTP Range头实现：
python复制headers = {}
if os.path.exists(local_path):
downloaded = os.path.getsize(local_path)
headers['Range'] = f'bytes={downloaded}-'
async with session.get(url, headers=headers) as response:
with open(local_path, 'ab') as f: # 追加模式
async for chunk in response.content.iter_chunked(1024):
f.write(chunk)
3. 完整实现与优化
3.1 核心下载逻辑
python复制async def _fetch(self, session, url):
try:
async with session.get(url, timeout=30) as response:
if response.status != 200:
raise DownloadError(f"HTTP {response.status}")
total = int(response.headers.get('content-length', 0))
progress = 0
with open(temp_path, 'wb') as f:
async for chunk in response.content.iter_chunked(8192):
f.write(chunk)
progress += len(chunk)
self._update_progress(url, progress, total)
os.rename(temp_path, final_path) # 原子操作
return final_path
except (aiohttp.ClientError, asyncio.TimeoutError) as e:
if os.path.exists(temp_path):
os.remove(temp_path)
raise
3.2 进度监控实现
使用回调函数实现解耦：
python复制def __init__(self, progress_callback=None):
self.progress_callback = progress_callback or self._default_progress
def _update_progress(self, url, progress, total):
speed = ... # 计算瞬时速度
self.progress_callback({
'url': url,
'progress': progress,
'total': total,
'speed': f"{speed/1024:.1f}KB/s",
'percentage': progress/total*100 if total else 0
})
3.3 速度限制算法
基于令牌桶算法实现：
python复制class RateLimiter:
def __init__(self, rate_kbps):
self.capacity = rate_kbps * 1024 # 转为bytes
self.tokens = self.capacity
self.last_check = time.monotonic()
async def consume(self, chunk_size):
now = time.monotonic()
elapsed = now - self.last_check
# 按时间补充令牌
self.tokens = min(
self.capacity,
self.tokens + elapsed * self.capacity
)
self.last_check = now
if self.tokens < chunk_size:
delay = (chunk_size - self.tokens) / self.capacity
await asyncio.sleep(delay)
self.tokens = 0
else:
self.tokens -= chunk_size
4. 实战技巧与排坑指南
4.1 必须关闭ClientSession
常见内存泄漏场景：
python复制# 错误示范：未关闭session
async def download_bad():
session = aiohttp.ClientSession()
await session.get(url)
# 忘记session.close()
# 正确做法
async def download_good():
async with aiohttp.ClientSession() as session:
await session.get(url)
4.2 DNS缓存优化
默认配置可能导致DNS查询成为瓶颈：
python复制connector = aiohttp.TCPConnector(
limit=0, # 不限制连接数（由Semaphore控制）
enable_cleanup_closed=True, # 自动清理关闭的连接
use_dns_cache=True, # 启用DNS缓存
ttl_dns_cache=300 # 缓存5分钟
)
4.3 错误重试策略
实现指数退避重试：
python复制async def download_with_retry(url, max_retries=3):
for attempt in range(max_retries):
try:
return await self.download(url)
except Exception as e:
if attempt == max_retries - 1:
raise
delay = min(2 ** attempt, 10) # 最大10秒
await asyncio.sleep(delay)
4.4 性能调优参数
关键配置建议值：
connector_limit: 建议设为并发数的1.5倍read_timeout: 大文件建议30-60秒connection_timeout: 建议5-10秒keepalive_timeout: 建议15秒
5. 完整示例代码
python复制import asyncio
import aiohttp
import os
from typing import Callable, Dict
class AsyncDownloader:
def __init__(self,
max_concurrent: int = 10,
rate_limit_kbps: int = 0,
progress_callback: Callable = None):
self.semaphore = asyncio.Semaphore(max_concurrent)
self.rate_limiter = RateLimiter(rate_limit_kbps) if rate_limit_kbps else None
self.progress_callback = progress_callback
async def download_file(self, url: str, save_path: str) -> str:
async with self.semaphore:
async with aiohttp.ClientSession() as session:
return await self._download(session, url, save_path)
async def _download(self, session, url, save_path):
temp_path = f"{save_path}.downloading"
try:
headers = {}
if os.path.exists(temp_path):
headers['Range'] = f'bytes={os.path.getsize(temp_path)}-'
async with session.get(url, headers=headers) as resp:
if resp.status == 416: # Range不满足
os.remove(temp_path)
return await self._download(session, url, save_path)
if resp.status not in (200, 206):
raise ValueError(f"HTTP {resp.status}")
mode = 'ab' if headers else 'wb'
total = int(resp.headers.get('content-length', 0))
downloaded = os.path.getsize(temp_path) if os.path.exists(temp_path) else 0
with open(temp_path, mode) as f:
async for chunk in resp.content.iter_chunked(8192):
if self.rate_limiter:
await self.rate_limiter.consume(len(chunk))
f.write(chunk)
downloaded += len(chunk)
if self.progress_callback:
self.progress_callback({
'url': url,
'downloaded': downloaded,
'total': total,
'speed': ... # 计算逻辑
})
os.rename(temp_path, save_path)
return save_path
except Exception:
if os.path.exists(temp_path):
os.remove(temp_path)
raise
async def batch_download(urls: Dict[str, str]):
"""urls: {url: save_path}"""
downloader = AsyncDownloader(max_concurrent=5)
tasks = [downloader.download_file(url, path)
for url, path in urls.items()]
return await asyncio.gather(*tasks, return_exceptions=True)
6. 进阶扩展方向
6.1 分布式扩展
结合消息队列实现分布式下载：
python复制async def consume_download_tasks(queue):
while True:
url, save_path = await queue.get()
try:
await downloader.download_file(url, save_path)
except Exception as e:
await queue.put((url, save_path)) # 重新入队
finally:
queue.task_done()
6.2 浏览器级并发控制
模拟浏览器连接行为：
python复制connector = aiohttp.TCPConnector(
limit_per_host=6, # 每个域名最大连接数
force_close=False,
enable_cleanup_closed=True
)
6.3 自适应限速算法
根据网络状况动态调整：
python复制def auto_adjust_speed(self, history_speeds):
# 基于过去30秒下载速度的P90值调整
if len(history_speeds) < 10:
return
target = sorted(history_speeds)[int(len(history_speeds)*0.9)]
self.rate_limiter.capacity = target * 0.8 # 保留余量
在实际项目中，这个下载器经过优化后可以达到单机500Mbps的下载吞吐量。关键是要根据具体场景调整并发数、chunk大小和缓冲区设置。对于特别大的文件（如视频），建议将chunk_size增加到32KB甚至64KB以减少上下文切换开销。