forked from Blendman974/faceclaw
Merge dedicated offline French Whisper model
This commit is contained in:
18 files changed
+621
-17
No files matched your search
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export the pinned French fine-tune to sherpa-onnx 1.13.0, using CPU only.
|
||||
|
||||
See docs/models/whisper-small-fr/README.md for the isolated export environment.
|
||||
This script never uploads assets or touches installed/downloaded phone models.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from types import SimpleNamespace
|
||||
import urllib.request
|
||||
|
||||
SOURCE_REVISION = '83d7df2928fb2218721c543dcc0be29ef8db2f96'
|
||||
SOURCE_BASE = f'https://huggingface.co/bofenghuang/whisper-small-cv11-french/resolve/{SOURCE_REVISION}/'
|
||||
WEIGHTS_SHA256 = '855a879fa3a71e75f29ed59fcfce287cf21bcdaab23530cd6f8c4ad192677afb'
|
||||
EXPORTER_URL = 'https://raw.githubusercontent.com/k2-fsa/sherpa-onnx/v1.13.0/scripts/whisper/export-onnx.py'
|
||||
EXPORTER_SHA256 = '5e0ddabfac8330ee9aecaba4c1b4286d1cd9f88df4cd30e54706b011bf976e45'
|
||||
MODEL_NAME = 'small-fr'
|
||||
|
||||
|
||||
def digest(path):
|
||||
with path.open('rb') as stream:
|
||||
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
|
||||
|
||||
def download(url, path, expected=None):
|
||||
if not path.exists():
|
||||
temp = path.with_suffix(path.suffix + '.part')
|
||||
urllib.request.urlretrieve(url, temp)
|
||||
temp.replace(path)
|
||||
if expected and digest(path) != expected:
|
||||
raise ValueError(f'Checksum mismatch: {path.name}')
|
||||
|
||||
|
||||
def openai_key(key):
|
||||
"""HF Whisper and OpenAI Whisper have identical tensors, differently named."""
|
||||
if not key.startswith('model.'):
|
||||
raise ValueError(f'Unexpected weight: {key}')
|
||||
key = key.removeprefix('model.')
|
||||
replacements = [
|
||||
('embed_positions.weight', 'positional_embedding'),
|
||||
('embed_tokens', 'token_embedding'), ('layers.', 'blocks.'),
|
||||
('self_attn_layer_norm', 'attn_ln'), ('encoder_attn_layer_norm', 'cross_attn_ln'),
|
||||
('final_layer_norm', 'mlp_ln'), ('self_attn', 'attn'), ('encoder_attn', 'cross_attn'),
|
||||
('k_proj', 'key'), ('q_proj', 'query'), ('v_proj', 'value'), ('out_proj', 'out'),
|
||||
('fc1', 'mlp.0'), ('fc2', 'mlp.2'),
|
||||
('encoder.layer_norm', 'encoder.ln_post'), ('decoder.layer_norm', 'decoder.ln'),
|
||||
]
|
||||
for old, new in replacements:
|
||||
key = key.replace(old, new)
|
||||
return key
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
output = args.output.resolve()
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
download(SOURCE_BASE + 'pytorch_model.bin', output / 'pytorch_model.bin', WEIGHTS_SHA256)
|
||||
download(SOURCE_BASE + 'config.json', output / 'config.json')
|
||||
download(EXPORTER_URL, output / 'sherpa-export.py', EXPORTER_SHA256)
|
||||
|
||||
import torch
|
||||
import whisper
|
||||
from whisper.model import ModelDimensions, disable_sdpa
|
||||
torch.set_num_threads(1)
|
||||
torch.set_num_interop_threads(1)
|
||||
torch.manual_seed(0)
|
||||
config = json.loads((output / 'config.json').read_text())
|
||||
dimensions = ModelDimensions(
|
||||
n_mels=config['num_mel_bins'], n_audio_ctx=config['max_source_positions'],
|
||||
n_audio_state=config['d_model'], n_audio_head=config['encoder_attention_heads'],
|
||||
n_audio_layer=config['encoder_layers'], n_vocab=config['vocab_size'],
|
||||
n_text_ctx=config['max_target_positions'], n_text_state=config['d_model'],
|
||||
n_text_head=config['decoder_attention_heads'], n_text_layer=config['decoder_layers'],
|
||||
)
|
||||
model = whisper.Whisper(dimensions)
|
||||
weights = torch.load(output / 'pytorch_model.bin', map_location='cpu', weights_only=True)
|
||||
converted = {openai_key(key): value for key, value in weights.items()}
|
||||
model.load_state_dict(converted, strict=True)
|
||||
del weights, converted
|
||||
model.eval()
|
||||
|
||||
spec = importlib.util.spec_from_file_location('sherpa_export', output / 'sherpa-export.py')
|
||||
exporter = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(exporter)
|
||||
exporter.get_args = lambda: SimpleNamespace(model=MODEL_NAME)
|
||||
exporter.load_model = lambda _name: model
|
||||
original_export = torch.onnx.export
|
||||
|
||||
def legacy_export(*values, **options):
|
||||
options['dynamo'] = False # sherpa 1.13.0 uses TorchScript's exporter.
|
||||
return original_export(*values, **options)
|
||||
|
||||
torch.onnx.export = legacy_export
|
||||
old_cwd = Path.cwd()
|
||||
try:
|
||||
os.chdir(output)
|
||||
with disable_sdpa():
|
||||
exporter.main()
|
||||
finally:
|
||||
torch.onnx.export = original_export
|
||||
os.chdir(old_cwd)
|
||||
|
||||
license_path = Path(__file__).resolve().parents[1] / 'docs/models/whisper-small-fr/LICENSE.txt'
|
||||
shutil.copyfile(license_path, output / 'LICENSE.txt')
|
||||
names = [f'{MODEL_NAME}-encoder.int8.onnx', f'{MODEL_NAME}-decoder.int8.onnx', f'{MODEL_NAME}-tokens.txt', 'LICENSE.txt']
|
||||
manifest = {
|
||||
'source': 'bofenghuang/whisper-small-cv11-french', 'revision': SOURCE_REVISION,
|
||||
'sourceSha256': WEIGHTS_SHA256, 'exporterSha256': EXPORTER_SHA256,
|
||||
'torchVersion': torch.__version__,
|
||||
'files': [{'name': name, 'sha256': digest(output / name), 'sizeBytes': (output / name).stat().st_size} for name in names],
|
||||
}
|
||||
(output / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
|
||||
print(json.dumps(manifest, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify released hashes and decode a public/test WAV with sherpa-onnx CPU."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import wave
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--model-dir', type=Path, required=True)
|
||||
parser.add_argument('--wav', type=Path, required=True)
|
||||
parser.add_argument('--expected', help='Optional expected transcription substring')
|
||||
args = parser.parse_args()
|
||||
model_dir = args.model_dir
|
||||
manifest = json.loads((model_dir / 'manifest.json').read_text())
|
||||
for entry in manifest['files']:
|
||||
path = model_dir / entry['name']
|
||||
with path.open('rb') as stream:
|
||||
actual = hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
if actual != entry['sha256'] or path.stat().st_size != entry['sizeBytes']:
|
||||
raise ValueError(f'Model file integrity check failed: {path.name}')
|
||||
|
||||
import numpy as np
|
||||
import sherpa_onnx
|
||||
with wave.open(str(args.wav)) as audio:
|
||||
if audio.getnchannels() != 1 or audio.getsampwidth() != 2:
|
||||
raise ValueError('Provide a mono PCM16 WAV')
|
||||
sample_rate = audio.getframerate()
|
||||
samples = np.frombuffer(audio.readframes(audio.getnframes()), dtype='<i2').astype(np.float32) / 32768
|
||||
recognizer = sherpa_onnx.OfflineRecognizer.from_whisper(
|
||||
encoder=str(model_dir / 'small-fr-encoder.int8.onnx'),
|
||||
decoder=str(model_dir / 'small-fr-decoder.int8.onnx'),
|
||||
tokens=str(model_dir / 'small-fr-tokens.txt'), num_threads=1,
|
||||
language='fr', task='transcribe',
|
||||
)
|
||||
stream = recognizer.create_stream()
|
||||
stream.accept_waveform(sample_rate, samples)
|
||||
start = time.monotonic()
|
||||
recognizer.decode_stream(stream)
|
||||
text = stream.result.text.strip()
|
||||
if not text or (args.expected and args.expected.casefold() not in text.casefold()):
|
||||
raise ValueError(f'Unexpected transcription: {text!r}')
|
||||
print(json.dumps({'sherpaVersion': sherpa_onnx.__version__, 'text': text,
|
||||
'audioSeconds': len(samples) / sample_rate,
|
||||
'decodeSeconds': time.monotonic() - start}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user