507 lines
13 KiB
Markdown
507 lines
13 KiB
Markdown
# Whisper语音识别
|
||
|
||
## 概述
|
||
|
||
Whisper是OpenAI于2022年发布的开源自动语音识别（ASR）系统，采用了基于Transformer的编解码器架构，能够进行多语言识别并具有较强的泛化能力。Whisper的核心创新在于使用弱监督训练方法，通过大规模互联网音频数据学习语音识别能力。
|
||
|
||
## 1. Whisper架构详解
|
||
|
||
### 1.1 编解码器架构
|
||
|
||
Whisper采用标准的Transformer编码器-解码器架构：
|
||
|
||
```python
|
||
class Whisper(nn.Module):
|
||
def __init__(self, config):
|
||
# 编码器
|
||
self.encoder = TransformerEncoder(
|
||
num_layers=config.num_layers,
|
||
d_model=config.d_model, # 1280
|
||
num_heads=config.num_heads, # 16
|
||
d_ff=config.d_ff, # 5120
|
||
dropout=0.1
|
||
)
|
||
|
||
# 解码器
|
||
self.decoder = TransformerDecoder(
|
||
num_layers=config.num_layers,
|
||
d_model=config.d_model,
|
||
num_heads=config.num_heads,
|
||
d_ff=config.d_ff,
|
||
dropout=0.1,
|
||
vocab_size=config.vocab_size # 51865
|
||
)
|
||
|
||
# 音频嵌入层
|
||
self.audio_embed = nn.Conv2d(
|
||
in_channels=80, # Mel频谱通道数
|
||
out_channels=config.d_model,
|
||
kernel_size=3,
|
||
stride=2
|
||
)
|
||
|
||
# 文本嵌入层
|
||
self.text_embed = nn.Embedding(
|
||
num_embeddings=config.vocab_size,
|
||
embedding_dim=config.d_model
|
||
)
|
||
```
|
||
|
||
### 1.2 音频特征处理
|
||
|
||
```python
|
||
def process_audio(audio, sample_rate=16000):
|
||
"""
|
||
音频预处理：转换为Mel频谱
|
||
"""
|
||
# 1. 重采样到16kHz
|
||
audio = resample(audio, target_sr=16000)
|
||
|
||
# 2. 短时傅里叶变换 (STFT)
|
||
stft = librosa.stft(audio, n_fft=400, hop_length=160)
|
||
|
||
# 3. 计算幅度谱
|
||
magnitude = np.abs(stft)
|
||
|
||
# 4. Mel滤波器组变换
|
||
mel_spec = librosa.feature.melspectrogram(
|
||
S=magnitude,
|
||
n_mels=80,
|
||
n_fft=400,
|
||
hop_length=160,
|
||
fmin=0,
|
||
fmax=8000
|
||
)
|
||
|
||
# 5. 对数压缩
|
||
log_mel = np.log(mel_spec + 1e-9)
|
||
|
||
# 6. 归一化
|
||
log_mel = (log_mel - log_mel.mean()) / log_mel.std()
|
||
|
||
return log_mel # [80, T] 其中T=audio_duration/0.01
|
||
```
|
||
|
||
### 1.3 编码器结构
|
||
|
||
```python
|
||
class WhisperEncoder(nn.Module):
|
||
def __init__(self, d_model=1280, num_layers=32):
|
||
# 输入卷积层
|
||
self.conv1 = nn.Conv1d(80, d_model, kernel_size=3, padding=1)
|
||
self.conv2 = nn.Conv1d(d_model, d_model, kernel_size=3, stride=2, padding=1)
|
||
|
||
# Transformer块
|
||
self.blocks = nn.ModuleList([
|
||
ResidualAttentionBlock(d_model, num_heads=16)
|
||
for _ in range(num_layers)
|
||
])
|
||
|
||
# 层归一化
|
||
self.ln = LayerNorm(d_model)
|
||
|
||
def forward(self, x):
|
||
# x: [B, 80, T]
|
||
|
||
# 初始卷积下采样
|
||
x = self.conv1(x)
|
||
x = nn.functional.gelu(x)
|
||
x = self.conv2(x)
|
||
x = nn.functional.gelu(x)
|
||
|
||
# Transformer处理
|
||
for block in self.blocks:
|
||
x = block(x, self_attn_mask=None)
|
||
|
||
# 最终归一化
|
||
x = self.ln(x)
|
||
|
||
return x # [B, D, T']
|
||
```
|
||
|
||
### 1.4 解码器结构
|
||
|
||
```python
|
||
class WhisperDecoder(nn.Module):
|
||
def __init__(self, d_model=1280, num_layers=32, vocab_size=51865):
|
||
# 文本嵌入
|
||
self.token_embedding = nn.Embedding(vocab_size, d_model)
|
||
|
||
# 位置编码
|
||
self.positional_embedding = nn.Embedding(5000, d_model)
|
||
|
||
# Transformer块
|
||
self.blocks = nn.ModuleList([
|
||
ResidualAttentionBlock(d_model, num_heads=16, use_cross_attention=True)
|
||
for _ in range(num_layers)
|
||
])
|
||
|
||
# 输出层
|
||
self.ln = LayerNorm(d_model)
|
||
self.proj = nn.Linear(d_model, vocab_size)
|
||
|
||
def forward(self, x, encoder_output):
|
||
# x: [B, seq_len] token IDs
|
||
# encoder_output: [B, D, T'] 编码器输出
|
||
|
||
# 文本嵌入 + 位置编码
|
||
x = self.token_embedding(x) + self.positional_embedding(
|
||
torch.arange(x.shape[1], device=x.device)
|
||
)
|
||
|
||
# 自注意力掩码（causal）
|
||
seq_len = x.shape[1]
|
||
mask = torch.triu(
|
||
torch.ones(seq_len, seq_len, device=x.device),
|
||
diagonal=1
|
||
).bool()
|
||
|
||
# Transformer块
|
||
for block in self.blocks:
|
||
x = block(x, encoder_output, self_attn_mask=mask)
|
||
|
||
# 输出投影
|
||
x = self.ln(x)
|
||
logits = self.proj(x)
|
||
|
||
return logits # [B, seq_len, vocab_size]
|
||
```
|
||
|
||
## 2. 多语言识别能力
|
||
|
||
### 2.1 支持的语言
|
||
|
||
Whisper能够识别99种语言：
|
||
|
||
```python
|
||
LANGUAGES = {
|
||
'en': 'english',
|
||
'zh': 'chinese',
|
||
'es': 'spanish',
|
||
'fr': 'french',
|
||
'de': 'german',
|
||
'ja': 'japanese',
|
||
'ko': 'korean',
|
||
'ar': 'arabic',
|
||
'ru': 'russian',
|
||
# ... 共99种语言
|
||
}
|
||
```
|
||
|
||
### 2.2 语言检测与自动切换
|
||
|
||
```python
|
||
def detect_language_and_transcribe(audio, model):
|
||
"""
|
||
自动检测语言并转录
|
||
"""
|
||
# 1. 音频特征提取
|
||
mel = process_audio(audio)
|
||
|
||
# 2. 语言检测（使用一小段音频）
|
||
lang_mel = mel[:, :3000] # 前30秒
|
||
lang_logits = model.detect_language(lang_mel)
|
||
detected_lang = LANGUAGES[lang_logits.argmax().item()]
|
||
|
||
# 3. 完整转录（指定语言）
|
||
if detected_lang != 'english':
|
||
# 使用语言特定的解码器
|
||
transcription = model.transcribe(
|
||
audio,
|
||
language=detected_lang,
|
||
task='transcribe'
|
||
)
|
||
else:
|
||
transcription = model.transcribe(audio)
|
||
|
||
return transcription
|
||
```
|
||
|
||
### 2.3 多语言训练数据
|
||
|
||
```python
|
||
# Whisper训练数据集分布
|
||
dataset_distribution = {
|
||
'english': 0.35, # 最大比例
|
||
'spanish': 0.10,
|
||
'chinese': 0.08,
|
||
'french': 0.07,
|
||
'german': 0.06,
|
||
'russian': 0.05,
|
||
'other': 0.29 # 其他语言混合
|
||
}
|
||
```
|
||
|
||
## 3. 弱监督训练方法
|
||
|
||
### 3.1 弱监督定义
|
||
|
||
**弱监督（Weak Supervision）：** 使用互联网上有噪声但数量庞大的音频-文本配对数据进行训练，相比使用人工标注的高质量数据，规模更大但标注质量较低。
|
||
|
||
```python
|
||
# 传统监督 vs 弱监督
|
||
traditional_supervision = {
|
||
'data_source': '人工标注',
|
||
'quality': '高',
|
||
'quantity': '低（数千到数万小时）',
|
||
'cost': '昂贵',
|
||
'annotation': '精确时间对齐'
|
||
}
|
||
|
||
weak_supervision = {
|
||
'data_source': '互联网音频（YouTube等）',
|
||
'quality': '中等（有噪声）',
|
||
'quantity': '大（数百万小时）',
|
||
'cost': '低',
|
||
'annotation': '不保证时间对齐'
|
||
}
|
||
```
|
||
|
||
### 3.2 训练数据收集
|
||
|
||
```python
|
||
def collect_weak_supervision_data():
|
||
"""
|
||
收集弱监督训练数据
|
||
"""
|
||
data_sources = [
|
||
'YouTube videos',
|
||
'Podcasts',
|
||
'Audiobooks',
|
||
'Public speaking recordings',
|
||
'Movie trailers',
|
||
'News broadcasts',
|
||
'Educational content'
|
||
]
|
||
|
||
collection_pipeline = {
|
||
'step1': '从公开源收集音频',
|
||
'step2': '原始转录文本获取（CC subtitles, transcripts）',
|
||
'step3': '质量过滤（长度、语言、清晰度）',
|
||
'step4': '音频-文本配对构建',
|
||
'step5': '去重（去除重复内容）'
|
||
}
|
||
|
||
return collected_data # Whisper使用约68万小时
|
||
```
|
||
|
||
### 3.3 数据预处理
|
||
|
||
```python
|
||
def preprocess_audio_text_pair(audio_path, text):
|
||
"""
|
||
预处理音频-文本对
|
||
"""
|
||
# 1. 音频清理
|
||
audio = load_audio(audio_path)
|
||
audio = remove_silence(audio)
|
||
audio = normalize_volume(audio)
|
||
|
||
# 2. 文本清理
|
||
text = clean_transcript(text)
|
||
text = remove_filler_words(text) # 可选
|
||
text = standardize_punctuation(text)
|
||
|
||
# 3. 时长过滤
|
||
if len(audio) < 0.5 or len(audio) > 30:
|
||
return None
|
||
|
||
# 4. 语言验证
|
||
detected_lang = detect_language(text)
|
||
if not is_supported_language(detected_lang):
|
||
return None
|
||
|
||
return {'audio': audio, 'text': text, 'language': detected_lang}
|
||
```
|
||
|
||
### 3.4 损失函数与训练
|
||
|
||
```python
|
||
def compute_loss(logits, targets, pad_token_id=50257):
|
||
"""
|
||
计算交叉熵损失
|
||
忽略padding token
|
||
"""
|
||
# logits: [B, seq_len, vocab_size]
|
||
# targets: [B, seq_len]
|
||
|
||
# 移位：解码器预测下一个token
|
||
shift_logits = logits[:, :-1, :].contiguous()
|
||
shift_targets = targets[:, 1:].contiguous()
|
||
|
||
# 计算交叉熵
|
||
loss = nn.functional.cross_entropy(
|
||
shift_logits.view(-1, shift_logits.size(-1)),
|
||
shift_targets.view(-1),
|
||
ignore_index=pad_token_id,
|
||
reduction='mean'
|
||
)
|
||
|
||
return loss
|
||
```
|
||
|
||
### 3.5 多任务训练
|
||
|
||
Whisper使用多任务学习框架：
|
||
|
||
```python
|
||
class MultiTaskWhisper(nn.Module):
|
||
def forward(self, audio, targets=None, task='transcribe', language='en'):
|
||
# 编码音频
|
||
encoder_output = self.encoder(audio)
|
||
|
||
# 构建任务提示
|
||
if task == 'transcribe':
|
||
task_tokens = [TO_TRANSCRIBE, LANG_TOKEN[language]]
|
||
elif task == 'translate':
|
||
task_tokens = [TO_TRANSLATE, LANG_TOKEN[language]]
|
||
elif task == 'detect_language':
|
||
task_tokens = [TO_DETECT_LANGUAGE]
|
||
|
||
# 解码
|
||
decoder_input = task_tokens + (targets or [])
|
||
decoder_output = self.decoder(decoder_input, encoder_output)
|
||
|
||
return decoder_output
|
||
```
|
||
|
||
## 4. 模型变体
|
||
|
||
### 4.1 Whisper模型系列
|
||
|
||
| 模型 | 参数量 | English WER | 多语言 WER |
|
||
|------|--------|-------------|------------|
|
||
| Tiny | 39M | ~8% | ~15% |
|
||
| Base | 74M | ~6% | ~12% |
|
||
| Small | 244M | ~4% | ~8% |
|
||
| Medium | 769M | ~3% | ~6% |
|
||
| Large-v2 | 1550M | ~2.5% | ~5% |
|
||
| Large-v3 | 1550M | ~2% | ~4% |
|
||
|
||
### 4.2 性能对比
|
||
|
||
```python
|
||
# Whisper vs 其他ASR系统
|
||
performance_comparison = {
|
||
'Whisper-Large-v3': {'WER': '2-3%', 'languages': 99, 'robustness': 'high'},
|
||
'Wav2Vec2': {'WER': '4-5%', 'languages': 50+', 'robustness': 'medium'},
|
||
'Google Speech-to-Text': {'WER': '3-4%', 'languages': 125+, 'robustness': 'high'},
|
||
'DeepSpeech': {'WER': '8-10%', 'languages': 5', 'robustness': 'low'}
|
||
}
|
||
```
|
||
|
||
## 5. 使用方法
|
||
|
||
### 5.1 基本推理
|
||
|
||
```python
|
||
import whisper
|
||
|
||
# 加载模型
|
||
model = whisper.load_model("base")
|
||
|
||
# 转录
|
||
result = model.transcribe("audio.mp3", language='zh')
|
||
|
||
# 打印结果
|
||
print(result["text"])
|
||
print(result["segments"])
|
||
```
|
||
|
||
### 5.2 多语言识别
|
||
|
||
```python
|
||
# 加载模型
|
||
model = whisper.load_model("medium")
|
||
|
||
# 方法1：自动检测语言
|
||
result = model.transcribe("unknown_language_audio.mp3")
|
||
|
||
# 方法2：指定语言
|
||
result = model.transcribe(
|
||
"japanese_audio.mp3",
|
||
language="ja",
|
||
task="transcribe" # 或 'translate'
|
||
)
|
||
|
||
# 方法3：翻译
|
||
result = model.transcribe(
|
||
"chinese_audio.mp3",
|
||
language="zh",
|
||
task="translate" # 翻译为英语
|
||
)
|
||
```
|
||
|
||
### 5.3 批量处理
|
||
|
||
```python
|
||
def batch_transcribe(audio_files, model, language=None):
|
||
"""
|
||
批量转录多个音频文件
|
||
"""
|
||
results = []
|
||
|
||
for audio_path in audio_files:
|
||
print(f"Processing: {audio_path}")
|
||
|
||
result = model.transcribe(
|
||
audio_path,
|
||
language=language,
|
||
verbose=False
|
||
)
|
||
|
||
results.append({
|
||
'file': audio_path,
|
||
'text': result['text'],
|
||
'language': result['language']
|
||
})
|
||
|
||
return results
|
||
```
|
||
|
||
## 6. 核心优势
|
||
|
||
| 优势 | 说明 |
|
||
|------|------|
|
||
| 多语言支持 | 99种语言，无需微调 |
|
||
| 弱监督训练 | 68万小时，规模大 |
|
||
| 鲁棒性 | 抗噪声能力强 |
|
||
| 开源 | 可本地部署 |
|
||
| 零样本能力 | 未经训练的语言也能识别 |
|
||
|
||
## 7. 局限性与改进
|
||
|
||
### 7.1 当前局限
|
||
|
||
| 限制 | 描述 |
|
||
|------|------|
|
||
| 实时性 | 推理延迟较高 |
|
||
| 专有名词 | 人名、地名识别准确率低 |
|
||
| 方言口音 | 某些方言识别困难 |
|
||
| 标点符号 | 自动标点不够准确 |
|
||
|
||
### 7.2 改进方向
|
||
|
||
```python
|
||
# Whisper的潜在改进
|
||
improvements = {
|
||
'speed': '优化推理速度，支持实时',
|
||
'accuracy': '提升专有名词识别',
|
||
'punctuation': '改进标点生成',
|
||
'dialect': '增加方言支持',
|
||
'speaker': '加入说话人分离'
|
||
}
|
||
```
|
||
|
||
## 8. 总结
|
||
|
||
Whisper的核心技术贡献：
|
||
|
||
| 技术 | 贡献 | 影响 |
|
||
|------|------|------|
|
||
| Transformer编解码器 | 统一的多语言ASR | 简化模型架构 |
|
||
| 弱监督训练 | 大规模互联网数据 | 降低数据成本 |
|
||
| 多语言泛化 | 99种语言零样本 | 广泛适用性 |
|
||
| 开源模型 | 可本地部署 | 推动社区发展 |
|
||
|
||
Whisper证明了弱监督训练在语音识别领域的有效性，为后续多语言语音模型的发展提供了重要参考。 |