import os
import random
from contextlib import contextmanager
from dataclasses import dataclass
from time import time
import torch
import torch.nn.functional as F
import torchaudio
from coqpit import Coqpit
from tqdm import tqdm
from TTS.tts.layers.tortoise.arch_utils import TorchMelSpectrogram
from TTS.tts.layers.tortoise.audio_utils import denormalize_tacotron_mel, load_voice, wav_to_univnet_mel
from TTS.tts.layers.tortoise.autoregressive import UnifiedVoice
from TTS.tts.layers.tortoise.classifier import AudioMiniEncoderWithClassifierHead
from TTS.tts.layers.tortoise.clvp import CLVP
from TTS.tts.layers.tortoise.diffusion import SpacedDiffusion, get_named_beta_schedule, space_timesteps
from TTS.tts.layers.tortoise.diffusion_decoder import DiffusionTts
from TTS.tts.layers.tortoise.random_latent_generator import RandomLatentConverter
from TTS.tts.layers.tortoise.tokenizer import VoiceBpeTokenizer
from TTS.tts.layers.tortoise.vocoder import VocConf, VocType
from TTS.tts.layers.tortoise.wav2vec_alignment import Wav2VecAlignment
from TTS.tts.models.base_tts import BaseTTS
def pad_or_truncate(t, length):
"""
Utility function for forcing <t> to have the specified sequence length, whether by clipping it or padding it with 0s.
"""
tp = t[..., :length]
if t.shape[-1] == length:
tp = t
elif t.shape[-1] < length:
tp = F.pad(t, (0, length - t.shape[-1]))
return tp
def deterministic_state(seed=None):
"""
Sets the random seeds that tortoise uses to the current time() and returns that seed so results can be
reproduced.
"""
seed = int(time()) if seed is None else seed
torch.manual_seed(seed)
random.seed(seed)
# Can't currently set this because of CUBLAS. TODO: potentially enable it if necessary.
# torch.use_deterministic_algorithms(True)
return seed
def load_discrete_vocoder_diffuser(
trained_diffusion_steps=4000,
desired_diffusion_steps=200,
cond_free=True,
cond_free_k=1,
sampler="ddim",
):
"""
Helper function to load a GaussianDiffusion instance configured for use as a vocoder.
"""
return SpacedDiffusion(
use_timesteps=space_timesteps(trained_diffusion_steps, [desired_diffusion_steps]),
model_mean_type="epsilon",
model_var_type="learned_range",
loss_type="mse",
betas=get_named_beta_schedule("linear", trained_diffusion_steps),
conditioning_free=cond_free,
conditioning_free_k=cond_free_k,
sampler=sampler,
)
def format_conditioning(clip, cond_length=132300, device="cuda", **kwargs):
"""
Converts the given conditioning signal to a MEL spectrogram and clips it as expected by the models.
"""
gap = clip.shape[-1] - cond_length
if gap < 0:
clip = F.pad(clip, pad=(0, abs(gap)))
elif gap > 0:
rand_start = random.randint(0, gap)
clip = clip[:, rand_start : rand_start + cond_length]
mel_clip = TorchMelSpectrogram(**kwargs)(clip.unsqueeze(0)).squeeze(0)
return mel_clip.unsqueeze(0).to(device)
def fix_autoregressive_output(codes, stop_token, complain=True):
"""
This function performs some padding on coded audio that fixes a mismatch issue between what the diffusion model was
trained on and what the autoregressive code generator creates (which has no padding or end).
This is highly specific to the DVAE being used, so this particular coding will not necessarily work if used with
a different DVAE. This can be inferred by feeding a audio clip padded with lots of zeros on the end through the DVAE
and copying out the last few codes.
Failing to do this padding will produce speech with a harsh end that sounds like "BLAH" or similar.
"""
# Strip off the autoregressive stop token and add padding.
stop_token_indices = (codes == stop_token).nonzero()
if len(stop_token_indices) == 0:
if complain:
print(
"No stop tokens found in one of the generated voice clips. This typically means the spoken audio is "
"too long. In some cases, the output will still be good, though. Listen to it and if it is missing words, "
"try breaking up your input text."
)
return codes
codes[stop_token_indices] = 83
stm = stop_token_indices.min().item()
codes[stm:] = 83
if stm - 3 < codes.shape[0]:
codes[-3] = 45
codes[-2] = 45
codes[-1] = 248
return codes
def do_spectrogram_diffusion(
diffusion_model,
diffuser,
latents,
conditioning_latents,
temperature=1,
verbose=True,
):
"""
Uses the specified diffusion model to convert discrete codes into a spectrogram.
"""
with torch.no_grad():
output_seq_len = (
latents.shape[1] * 4 * 24000 // 22050
) # This diffusion model converts from 22kHz spectrogram codes to a 24kHz spectrogram signal.
output_shape = (latents.shape[0], 100, output_seq_len)
precomputed_embeddings = diffusion_model.timestep_independent(
latents, conditioning_latents, output_seq_len, False
)
noise = torch.randn(output_shape, device=latents.device) * temperature
mel = diffuser.sample_loop(
diffusion_model,
output_shape,
noise=noise,
model_kwargs={"precomputed_aligned_embeddings": precomputed_embeddings},
progress=verbose,
)
return denormalize_tacotron_mel(mel)[:, :, :output_seq_len]
def classify_audio_clip(clip, model_dir):
"""
Returns whether or not Tortoises' classifier thinks the given clip came from Tortoise.
:param clip: torch tensor containing audio waveform data (get it from load_audio)
:return: True if the clip was classified as coming from Tortoise and false if it was classified as real.
"""
classifier = AudioMiniEncoderWithClassifierHead(
2,
spec_dim=1,
embedding_dim=512,
depth=5,
downsample_factor=4,
resnet_blocks=2,
attn_blocks=4,
num_attn_heads=4,
base_channels=32,
dropout=0,
kernel_size=5,
distribute_zero_label=False,
)
classifier.load_state_dict(torch.load(os.path.join(model_dir, "classifier.pth"), map_location=torch.device("cpu")))
clip = clip.cpu().unsqueeze(0)
results = F.softmax(classifier(clip), dim=-1)
return results[0][0]
def pick_best_batch_size_for_gpu():
"""
Tries to pick a batch size that will fit in your GPU. These sizes aren't guaranteed to work, but they should give
you a good shot.
"""
if torch.cuda.is_available():
_, available = torch.cuda.mem_get_info()
availableGb = available / (1024**3)
batch_size = 1
if availableGb > 14:
batch_size = 16
elif availableGb > 10:
batch_size = 8
elif availableGb > 7:
batch_size = 4
return batch_size
@dataclass
class TortoiseAudioConfig(Coqpit):
sample_rate: int = 22050
diffusion_sample_rate: int = 24000
output_sample_rate: int = 24000
[docs]
@dataclass
class TortoiseArgs(Coqpit):
"""A dataclass to represent Tortoise model arguments that define the model structure.
Args:
autoregressive_batch_size (int): The size of the auto-regressive batch.
enable_redaction (bool, optional): Whether to enable redaction. Defaults to True.
high_vram (bool, optional): Whether to use high VRAM. Defaults to False.
kv_cache (bool, optional): Whether to use the kv_cache. Defaults to True.
ar_checkpoint (str, optional): The checkpoint for the autoregressive model. Defaults to None.
clvp_checkpoint (str, optional): The checkpoint for the ConditionalLatentVariablePerseq model. Defaults to None.
diff_checkpoint (str, optional): The checkpoint for the DiffTTS model. Defaults to None.
num_chars (int, optional): The maximum number of characters to generate. Defaults to 255.
vocoder (VocType, optional): The vocoder to use for synthesis. Defaults to VocConf.Univnet.
For UnifiedVoice model:
ar_max_mel_tokens (int, optional): The maximum mel tokens for the autoregressive model. Defaults to 604.
ar_max_text_tokens (int, optional): The maximum text tokens for the autoregressive model. Defaults to 402.
ar_max_conditioning_inputs (int, optional): The maximum conditioning inputs for the autoregressive model. Defaults to 2.
ar_layers (int, optional): The number of layers for the autoregressive model. Defaults to 30.
ar_model_dim (int, optional): The model dimension for the autoregressive model. Defaults to 1024.
ar_heads (int, optional): The number of heads for the autoregressive model. Defaults to 16.
ar_number_text_tokens (int, optional): The number of text tokens for the autoregressive model. Defaults to 255.
ar_start_text_token (int, optional): The start text token for the autoregressive model. Defaults to 255.
ar_checkpointing (bool, optional): Whether to use checkpointing for the autoregressive model. Defaults to False.
ar_train_solo_embeddings (bool, optional): Whether to train embeddings for the autoregressive model. Defaults to False.
For DiffTTS model:
diff_model_channels (int, optional): The number of channels for the DiffTTS model. Defaults to 1024.
diff_num_layers (int, optional): The number of layers for the DiffTTS model. Defaults to 10.
diff_in_channels (int, optional): The input channels for the DiffTTS model. Defaults to 100.
diff_out_channels (int, optional): The output channels for the DiffTTS model. Defaults to 200.
diff_in_latent_channels (int, optional): The input latent channels for the DiffTTS model. Defaults to 1024.
diff_in_tokens (int, optional): The input tokens for the DiffTTS model. Defaults to 8193.
diff_dropout (int, optional): The dropout percentage for the DiffTTS model. Defaults to 0.
diff_use_fp16 (bool, optional): Whether to use fp16 for the DiffTTS model. Defaults to False.
diff_num_heads (int, optional): The number of heads for the DiffTTS model. Defaults to 16.
diff_layer_drop (int, optional): The layer dropout percentage for the DiffTTS model. Defaults to 0.
diff_unconditioned_percentage (int, optional): The percentage of unconditioned inputs for the DiffTTS model. Defaults to 0.
For ConditionalLatentVariablePerseq model:
clvp_dim_text (int): The dimension of the text input for the CLVP module. Defaults to 768.
clvp_dim_speech (int): The dimension of the speech input for the CLVP module. Defaults to 768.
clvp_dim_latent (int): The dimension of the latent representation for the CLVP module. Defaults to 768.
clvp_num_text_tokens (int): The number of text tokens used by the CLVP module. Defaults to 256.
clvp_text_enc_depth (int): The depth of the text encoder in the CLVP module. Defaults to 20.
clvp_text_seq_len (int): The maximum sequence length of the text input for the CLVP module. Defaults to 350.
clvp_text_heads (int): The number of attention heads used by the text encoder in the CLVP module. Defaults to 12.
clvp_num_speech_tokens (int): The number of speech tokens used by the CLVP module. Defaults to 8192.
clvp_speech_enc_depth (int): The depth of the speech encoder in the CLVP module. Defaults to 20.
clvp_speech_heads (int): The number of attention heads used by the speech encoder in the CLVP module. Defaults to 12.
clvp_speech_seq_len (int): The maximum sequence length of the speech input for the CLVP module. Defaults to 430.
clvp_use_xformers (bool): A flag indicating whether the model uses transformers in the CLVP module. Defaults to True.
duration_const (int): A constant value used in the model. Defaults to 102400.
"""
autoregressive_batch_size: int = 1
enable_redaction: bool = False
high_vram: bool = False
kv_cache: bool = True
ar_checkpoint: str = None
clvp_checkpoint: str = None
diff_checkpoint: str = None
num_chars: int = 255
vocoder: VocType = VocConf.Univnet
# UnifiedVoice params
ar_max_mel_tokens: int = 604
ar_max_text_tokens: int = 402
ar_max_conditioning_inputs: int = 2
ar_layers: int = 30
ar_model_dim: int = 1024
ar_heads: int = 16
ar_number_text_tokens: int = 255
ar_start_text_token: int = 255
ar_checkpointing: bool = False
ar_train_solo_embeddings: bool = False
# DiffTTS params
diff_model_channels: int = 1024
diff_num_layers: int = 10
diff_in_channels: int = 100
diff_out_channels: int = 200
diff_in_latent_channels: int = 1024
diff_in_tokens: int = 8193
diff_dropout: int = 0
diff_use_fp16: bool = False
diff_num_heads: int = 16
diff_layer_drop: int = 0
diff_unconditioned_percentage: int = 0
# clvp params
clvp_dim_text: int = 768
clvp_dim_speech: int = 768
clvp_dim_latent: int = 768
clvp_num_text_tokens: int = 256
clvp_text_enc_depth: int = 20
clvp_text_seq_len: int = 350
clvp_text_heads: int = 12
clvp_num_speech_tokens: int = 8192
clvp_speech_enc_depth: int = 20
clvp_speech_heads: int = 12
clvp_speech_seq_len: int = 430
clvp_use_xformers: bool = True
# constants
duration_const: int = 102400
[docs]
class Tortoise(BaseTTS):
"""Tortoise model class.
Currently only supports inference.
Examples:
>>> from TTS.tts.configs.tortoise_config import TortoiseConfig
>>> from TTS.tts.models.tortoise import Tortoise
>>> config = TortoiseConfig()
>>> model = Tortoise.inif_from_config(config)
>>> model.load_checkpoint(config, checkpoint_dir="paths/to/models_dir/", eval=True)
"""
def __init__(self, config: Coqpit):
super().__init__(config, ap=None, tokenizer=None)
self.mel_norm_path = None
self.config = config
self.ar_checkpoint = self.args.ar_checkpoint
self.diff_checkpoint = self.args.diff_checkpoint # TODO: check if this is even needed
self.models_dir = config.model_dir
self.autoregressive_batch_size = (
pick_best_batch_size_for_gpu()
if self.args.autoregressive_batch_size is None
else self.args.autoregressive_batch_size
)
self.enable_redaction = self.args.enable_redaction
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if self.enable_redaction:
self.aligner = Wav2VecAlignment()
self.tokenizer = VoiceBpeTokenizer()
self.autoregressive = UnifiedVoice(
max_mel_tokens=self.args.ar_max_mel_tokens,
max_text_tokens=self.args.ar_max_text_tokens,
max_conditioning_inputs=self.args.ar_max_conditioning_inputs,
layers=self.args.ar_layers,
model_dim=self.args.ar_model_dim,
heads=self.args.ar_heads,
number_text_tokens=self.args.ar_number_text_tokens,
start_text_token=self.args.ar_start_text_token,
checkpointing=self.args.ar_checkpointing,
train_solo_embeddings=self.args.ar_train_solo_embeddings,
).cpu()
self.diffusion = DiffusionTts(
model_channels=self.args.diff_model_channels,
num_layers=self.args.diff_num_layers,
in_channels=self.args.diff_in_channels,
out_channels=self.args.diff_out_channels,
in_latent_channels=self.args.diff_in_latent_channels,
in_tokens=self.args.diff_in_tokens,
dropout=self.args.diff_dropout,
use_fp16=self.args.diff_use_fp16,
num_heads=self.args.diff_num_heads,
layer_drop=self.args.diff_layer_drop,
unconditioned_percentage=self.args.diff_unconditioned_percentage,
).cpu()
self.clvp = CLVP(
dim_text=self.args.clvp_dim_text,
dim_speech=self.args.clvp_dim_speech,
dim_latent=self.args.clvp_dim_latent,
num_text_tokens=self.args.clvp_num_text_tokens,
text_enc_depth=self.args.clvp_text_enc_depth,
text_seq_len=self.args.clvp_text_seq_len,
text_heads=self.args.clvp_text_heads,
num_speech_tokens=self.args.clvp_num_speech_tokens,
speech_enc_depth=self.args.clvp_speech_enc_depth,
speech_heads=self.args.clvp_speech_heads,
speech_seq_len=self.args.clvp_speech_seq_len,
use_xformers=self.args.clvp_use_xformers,
).cpu()
self.vocoder = self.args.vocoder.value.constructor().cpu()
# Random latent generators (RLGs) are loaded lazily.
self.rlg_auto = None
self.rlg_diffusion = None
if self.args.high_vram:
self.autoregressive = self.autoregressive.to(self.device)
self.diffusion = self.diffusion.to(self.device)
self.clvp = self.clvp.to(self.device)
self.vocoder = self.vocoder.to(self.device)
self.high_vram = self.args.high_vram
@contextmanager
def temporary_cuda(self, model):
if self.high_vram:
yield model
else:
m = model.to(self.device)
yield m
m = model.cpu()
[docs]
def get_conditioning_latents(
self,
voice_samples,
return_mels=False,
latent_averaging_mode=0,
original_tortoise=False,
):
"""
Transforms one or more voice_samples into a tuple (autoregressive_conditioning_latent, diffusion_conditioning_latent).
These are expressive learned latents that encode aspects of the provided clips like voice, intonation, and acoustic
properties.
:param voice_samples: List of arbitrary reference clips, which should be *pairs* of torch tensors containing arbitrary kHz waveform data.
:param latent_averaging_mode: 0/1/2 for following modes:
0 - latents will be generated as in original tortoise, using ~4.27s from each voice sample, averaging latent across all samples
1 - latents will be generated using (almost) entire voice samples, averaged across all the ~4.27s chunks
2 - latents will be generated using (almost) entire voice samples, averaged per voice sample
"""
assert latent_averaging_mode in [
0,
1,
2,
], "latent_averaging mode has to be one of (0, 1, 2)"
with torch.no_grad():
voice_samples = [[v.to(self.device) for v in ls] for ls in voice_samples]
auto_conds = []
for ls in voice_samples:
auto_conds.append(format_conditioning(ls[0], device=self.device, mel_norm_file=self.mel_norm_path))
auto_conds = torch.stack(auto_conds, dim=1)
with self.temporary_cuda(self.autoregressive) as ar:
auto_latent = ar.get_conditioning(auto_conds)
diffusion_conds = []
DURS_CONST = self.args.duration_const
for ls in voice_samples:
# The diffuser operates at a sample rate of 24000 (except for the latent inputs)
sample = torchaudio.functional.resample(ls[0], 22050, 24000) if original_tortoise else ls[1]
if latent_averaging_mode == 0:
sample = pad_or_truncate(sample, DURS_CONST)
cond_mel = wav_to_univnet_mel(
sample.to(self.device),
do_normalization=False,
device=self.device,
)
diffusion_conds.append(cond_mel)
else:
from math import ceil
if latent_averaging_mode == 2:
temp_diffusion_conds = []
for chunk in range(ceil(sample.shape[1] / DURS_CONST)):
current_sample = sample[:, chunk * DURS_CONST : (chunk + 1) * DURS_CONST]
current_sample = pad_or_truncate(current_sample, DURS_CONST)
cond_mel = wav_to_univnet_mel(
current_sample.to(self.device),
do_normalization=False,
device=self.device,
)
if latent_averaging_mode == 1:
diffusion_conds.append(cond_mel)
elif latent_averaging_mode == 2:
temp_diffusion_conds.append(cond_mel)
if latent_averaging_mode == 2:
diffusion_conds.append(torch.stack(temp_diffusion_conds).mean(0))
diffusion_conds = torch.stack(diffusion_conds, dim=1)
with self.temporary_cuda(self.diffusion) as diffusion:
diffusion_latent = diffusion.get_conditioning(diffusion_conds)
if return_mels:
return auto_latent, diffusion_latent, auto_conds, diffusion_conds
return auto_latent, diffusion_latent
def get_random_conditioning_latents(self):
# Lazy-load the RLG models.
if self.rlg_auto is None:
self.rlg_auto = RandomLatentConverter(1024).eval()
self.rlg_auto.load_state_dict(
torch.load(
os.path.join(self.models_dir, "rlg_auto.pth"),
map_location=torch.device("cpu"),
)
)
self.rlg_diffusion = RandomLatentConverter(2048).eval()
self.rlg_diffusion.load_state_dict(
torch.load(
os.path.join(self.models_dir, "rlg_diffuser.pth"),
map_location=torch.device("cpu"),
)
)
with torch.no_grad():
return self.rlg_auto(torch.tensor([0.0])), self.rlg_diffusion(torch.tensor([0.0]))
[docs]
def synthesize(self, text, config, speaker_id="random", voice_dirs=None, **kwargs):
"""Synthesize speech with the given input text.
Args:
text (str): Input text.
config (TortoiseConfig): Config with inference parameters.
speaker_id (str): One of the available speaker names. If `random`, it generates a random speaker.
voice_dirs (List[str]): List of paths that host reference audio files for speakers. Defaults to None.
**kwargs: Inference settings. See `inference()`.
Returns:
A dictionary of the output values with `wav` as output waveform, `deterministic_seed` as seed used at inference,
`text_input` as text token IDs after tokenizer, `voice_samples` as samples used for cloning, `conditioning_latents`
as latents used at inference.
"""
speaker_id = "random" if speaker_id is None else speaker_id
if voice_dirs is not None:
voice_dirs = [voice_dirs]
voice_samples, conditioning_latents = load_voice(speaker_id, voice_dirs)
else:
voice_samples, conditioning_latents = load_voice(speaker_id)
outputs = self.inference_with_config(
text, config, voice_samples=voice_samples, conditioning_latents=conditioning_latents, **kwargs
)
return_dict = {
"wav": outputs["wav"],
"deterministic_seed": outputs["deterministic_seed"],
"text_inputs": outputs["text"],
"voice_samples": outputs["voice_samples"],
"conditioning_latents": outputs["conditioning_latents"],
}
return return_dict
[docs]
def inference_with_config(self, text, config, **kwargs):
"""
inference with config
#TODO describe in detail
"""
# Use generally found best tuning knobs for generation.
settings = {
"temperature": config.temperature,
"length_penalty": config.length_penalty,
"repetition_penalty": config.repetition_penalty,
"top_p": config.top_p,
"cond_free_k": config.cond_free_k,
"diffusion_temperature": config.diffusion_temperature,
"sampler": config.sampler,
}
# Presets are defined here.
presets = {
"single_sample": {
"num_autoregressive_samples": 8,
"diffusion_iterations": 10,
"sampler": "ddim",
},
"ultra_fast": {
"num_autoregressive_samples": 16,
"diffusion_iterations": 10,
"sampler": "ddim",
},
"ultra_fast_old": {
"num_autoregressive_samples": 16,
"diffusion_iterations": 30,
"cond_free": False,
},
"very_fast": {
"num_autoregressive_samples": 32,
"diffusion_iterations": 30,
"sampler": "dpm++2m",
},
"fast": {
"num_autoregressive_samples": 5,
"diffusion_iterations": 50,
"sampler": "ddim",
},
"fast_old": {"num_autoregressive_samples": 96, "diffusion_iterations": 80},
"standard": {
"num_autoregressive_samples": 5,
"diffusion_iterations": 200,
},
"high_quality": {
"num_autoregressive_samples": 256,
"diffusion_iterations": 400,
},
}
if "preset" in kwargs:
settings.update(presets[kwargs["preset"]])
kwargs.pop("preset")
settings.update(kwargs) # allow overriding of preset settings with kwargs
return self.inference(text, **settings)
[docs]
def inference(
self,
text,
voice_samples=None,
conditioning_latents=None,
k=1,
verbose=True,
use_deterministic_seed=None,
return_deterministic_state=False,
latent_averaging_mode=0,
# autoregressive generation parameters follow
num_autoregressive_samples=16,
temperature=0.8,
length_penalty=1,
repetition_penalty=2.0,
top_p=0.8,
max_mel_tokens=500,
# diffusion generation parameters follow
diffusion_iterations=100,
cond_free=True,
cond_free_k=2,
diffusion_temperature=1.0,
sampler="ddim",
half=True,
original_tortoise=False,
**hf_generate_kwargs,
):
"""
This function produces an audio clip of the given text being spoken with the given reference voice.
Args:
text: (str) Text to be spoken.
voice_samples: (List[Tuple[torch.Tensor]]) List of an arbitrary number of reference clips, which should be tuple-pairs
of torch tensors containing arbitrary kHz waveform data.
conditioning_latents: (Tuple[autoregressive_conditioning_latent, diffusion_conditioning_latent]) A tuple of
(autoregressive_conditioning_latent, diffusion_conditioning_latent), which can be provided in lieu
of voice_samples. This is ignored unless `voice_samples=None`. Conditioning latents can be retrieved
via `get_conditioning_latents()`.
k: (int) The number of returned clips. The most likely (as determined by Tortoises' CLVP model) clips are returned.
latent_averaging_mode: (int) 0/1/2 for following modes:
0 - latents will be generated as in original tortoise, using ~4.27s from each voice sample, averaging latent across all samples
1 - latents will be generated using (almost) entire voice samples, averaged across all the ~4.27s chunks
2 - latents will be generated using (almost) entire voice samples, averaged per voice sample
verbose: (bool) Whether or not to print log messages indicating the progress of creating a clip. Default=true.
num_autoregressive_samples: (int) Number of samples taken from the autoregressive model, all of which are filtered using CLVP.
As Tortoise is a probabilistic model, more samples means a higher probability of creating something "great".
temperature: (float) The softmax temperature of the autoregressive model.
length_penalty: (float) A length penalty applied to the autoregressive decoder. Higher settings causes the model to produce more terse outputs.
repetition_penalty: (float) A penalty that prevents the autoregressive decoder from repeating itself during decoding. Can be used to reduce
the incidence of long silences or "uhhhhhhs", etc.
top_p: (float) P value used in nucleus sampling. (0,1]. Lower values mean the decoder produces more "likely" (aka boring) outputs.
max_mel_tokens: (int) Restricts the output length. (0,600] integer. Each unit is 1/20 of a second.
typical_sampling: (bool) Turns typical sampling on or off. This sampling mode is discussed in this paper: https://arxiv.org/abs/2202.00666
I was interested in the premise, but the results were not as good as I was hoping. This is off by default, but could use some tuning.
typical_mass: (float) The typical_mass parameter from the typical_sampling algorithm.
diffusion_iterations: (int) Number of diffusion steps to perform. [0,4000]. More steps means the network has more chances to iteratively
refine the output, which should theoretically mean a higher quality output. Generally a value above 250 is not noticeably better, however.
cond_free: (bool) Whether or not to perform conditioning-free diffusion. Conditioning-free diffusion performs two forward passes for
each diffusion step: one with the outputs of the autoregressive model and one with no conditioning priors. The output of the two
is blended according to the cond_free_k value below. Conditioning-free diffusion is the real deal, and dramatically improves realism.
cond_free_k: (float) Knob that determines how to balance the conditioning free signal with the conditioning-present signal. [0,inf].
As cond_free_k increases, the output becomes dominated by the conditioning-free signal.
diffusion_temperature: (float) Controls the variance of the noise fed into the diffusion model. [0,1]. Values at 0
are the "mean" prediction of the diffusion network and will sound bland and smeared.
hf_generate_kwargs: (**kwargs) The huggingface Transformers generate API is used for the autoregressive transformer.
Extra keyword args fed to this function get forwarded directly to that API. Documentation
here: https://huggingface.co/docs/transformers/internal/generation_utils
Returns:
Generated audio clip(s) as a torch tensor. Shape 1,S if k=1 else, (k,1,S) where S is the sample length.
Sample rate is 24kHz.
"""
deterministic_seed = deterministic_state(seed=use_deterministic_seed)
text_tokens = torch.IntTensor(self.tokenizer.encode(text)).unsqueeze(0).to(self.device)
text_tokens = F.pad(text_tokens, (0, 1)) # This may not be necessary.
assert (
text_tokens.shape[-1] < 400
), "Too much text provided. Break the text up into separate segments and re-try inference."
if voice_samples is not None:
(
auto_conditioning,
diffusion_conditioning,
_,
_,
) = self.get_conditioning_latents(
voice_samples,
return_mels=True,
latent_averaging_mode=latent_averaging_mode,
original_tortoise=original_tortoise,
)
elif conditioning_latents is not None:
auto_conditioning, diffusion_conditioning = conditioning_latents
else:
(
auto_conditioning,
diffusion_conditioning,
) = self.get_random_conditioning_latents()
auto_conditioning = auto_conditioning.to(self.device)
diffusion_conditioning = diffusion_conditioning.to(self.device)
diffuser = load_discrete_vocoder_diffuser(
desired_diffusion_steps=diffusion_iterations, cond_free=cond_free, cond_free_k=cond_free_k, sampler=sampler
)
# in the case of single_sample,
orig_batch_size = self.autoregressive_batch_size
while num_autoregressive_samples % self.autoregressive_batch_size:
self.autoregressive_batch_size //= 2
with torch.no_grad():
samples = []
num_batches = num_autoregressive_samples // self.autoregressive_batch_size
stop_mel_token = self.autoregressive.stop_mel_token
calm_token = (
83 # This is the token for coding silence, which is fixed in place with "fix_autoregressive_output"
)
self.autoregressive = self.autoregressive.to(self.device)
if verbose:
print("Generating autoregressive samples..")
with self.temporary_cuda(self.autoregressive) as autoregressive, torch.autocast(
device_type="cuda", dtype=torch.float16, enabled=half
):
for b in tqdm(range(num_batches), disable=not verbose):
codes = autoregressive.inference_speech(
auto_conditioning,
text_tokens,
do_sample=True,
top_p=top_p,
temperature=temperature,
num_return_sequences=self.autoregressive_batch_size,
length_penalty=length_penalty,
repetition_penalty=repetition_penalty,
max_generate_length=max_mel_tokens,
**hf_generate_kwargs,
)
padding_needed = max_mel_tokens - codes.shape[1]
codes = F.pad(codes, (0, padding_needed), value=stop_mel_token)
samples.append(codes)
self.autoregressive_batch_size = orig_batch_size # in the case of single_sample
clip_results = []
with self.temporary_cuda(self.clvp) as clvp, torch.autocast(
device_type="cuda", dtype=torch.float16, enabled=half
):
for batch in tqdm(samples, disable=not verbose):
for i in range(batch.shape[0]):
batch[i] = fix_autoregressive_output(batch[i], stop_mel_token)
clvp_res = clvp(
text_tokens.repeat(batch.shape[0], 1),
batch,
return_loss=False,
)
clip_results.append(clvp_res)
clip_results = torch.cat(clip_results, dim=0)
samples = torch.cat(samples, dim=0)
best_results = samples[torch.topk(clip_results, k=k).indices]
del samples
# The diffusion model actually wants the last hidden layer from the autoregressive model as conditioning
# inputs. Re-produce those for the top results. This could be made more efficient by storing all of these
# results, but will increase memory usage.
with self.temporary_cuda(self.autoregressive) as autoregressive:
best_latents = autoregressive(
auto_conditioning.repeat(k, 1),
text_tokens.repeat(k, 1),
torch.tensor([text_tokens.shape[-1]], device=text_tokens.device),
best_results,
torch.tensor(
[best_results.shape[-1] * self.autoregressive.mel_length_compression],
device=text_tokens.device,
),
return_latent=True,
clip_inputs=False,
)
del auto_conditioning
if verbose:
print("Transforming autoregressive outputs into audio..")
wav_candidates = []
for b in range(best_results.shape[0]):
codes = best_results[b].unsqueeze(0)
latents = best_latents[b].unsqueeze(0)
# Find the first occurrence of the "calm" token and trim the codes to that.
ctokens = 0
for code in range(codes.shape[-1]):
if codes[0, code] == calm_token:
ctokens += 1
else:
ctokens = 0
if ctokens > 8: # 8 tokens gives the diffusion model some "breathing room" to terminate speech.
latents = latents[:, :code]
break
with self.temporary_cuda(self.diffusion) as diffusion:
mel = do_spectrogram_diffusion(
diffusion,
diffuser,
latents,
diffusion_conditioning,
temperature=diffusion_temperature,
verbose=verbose,
)
with self.temporary_cuda(self.vocoder) as vocoder:
wav = vocoder.inference(mel)
wav_candidates.append(wav.cpu())
def potentially_redact(clip, text):
if self.enable_redaction:
return self.aligner.redact(clip.squeeze(1), text).unsqueeze(1)
return clip
wav_candidates = [potentially_redact(wav_candidate, text) for wav_candidate in wav_candidates]
if len(wav_candidates) > 1:
res = wav_candidates
else:
res = wav_candidates[0]
return_dict = {
"wav": res,
"deterministic_seed": None,
"text": None,
"voice_samples": None,
"conditioning_latents": None,
}
if return_deterministic_state:
return_dict = {
"wav": res,
"deterministic_seed": deterministic_seed,
"text": text,
"voice_samples": voice_samples,
"conditioning_latents": conditioning_latents,
}
return return_dict
def forward(self):
raise NotImplementedError("Tortoise Training is not implemented")
def eval_step(self):
raise NotImplementedError("Tortoise Training is not implemented")
@staticmethod
def init_from_config(config: "TortoiseConfig", **kwargs): # pylint: disable=unused-argument
return Tortoise(config)
[docs]
def load_checkpoint(
self,
config,
checkpoint_dir,
ar_checkpoint_path=None,
diff_checkpoint_path=None,
clvp_checkpoint_path=None,
vocoder_checkpoint_path=None,
eval=False,
strict=True,
**kwargs,
): # pylint: disable=unused-argument, redefined-builtin
"""Load a model checkpoints from a directory. This model is with multiple checkpoint files and it
expects to have all the files to be under the given `checkpoint_dir` with the rigth names.
If eval is True, set the model to eval mode.
Args:
config (TortoiseConfig): The model config.
checkpoint_dir (str): The directory where the checkpoints are stored.
ar_checkpoint_path (str, optional): The path to the autoregressive checkpoint. Defaults to None.
diff_checkpoint_path (str, optional): The path to the diffusion checkpoint. Defaults to None.
clvp_checkpoint_path (str, optional): The path to the CLVP checkpoint. Defaults to None.
vocoder_checkpoint_path (str, optional): The path to the vocoder checkpoint. Defaults to None.
eval (bool, optional): Whether to set the model to eval mode. Defaults to False.
strict (bool, optional): Whether to load the model strictly. Defaults to True.
"""
if self.models_dir is None:
self.models_dir = checkpoint_dir
ar_path = ar_checkpoint_path or os.path.join(checkpoint_dir, "autoregressive.pth")
diff_path = diff_checkpoint_path or os.path.join(checkpoint_dir, "diffusion_decoder.pth")
clvp_path = clvp_checkpoint_path or os.path.join(checkpoint_dir, "clvp2.pth")
vocoder_checkpoint_path = vocoder_checkpoint_path or os.path.join(checkpoint_dir, "vocoder.pth")
self.mel_norm_path = os.path.join(checkpoint_dir, "mel_norms.pth")
if os.path.exists(ar_path):
# remove keys from the checkpoint that are not in the model
checkpoint = torch.load(ar_path, map_location=torch.device("cpu"))
# strict set False
# due to removed `bias` and `masked_bias` changes in Transformers
self.autoregressive.load_state_dict(checkpoint, strict=False)
if os.path.exists(diff_path):
self.diffusion.load_state_dict(torch.load(diff_path), strict=strict)
if os.path.exists(clvp_path):
self.clvp.load_state_dict(torch.load(clvp_path), strict=strict)
if os.path.exists(vocoder_checkpoint_path):
self.vocoder.load_state_dict(
config.model_args.vocoder.value.optionally_index(
torch.load(
vocoder_checkpoint_path,
map_location=torch.device("cpu"),
)
)
)
if eval:
self.autoregressive.post_init_gpt2_config(self.args.kv_cache)
self.autoregressive.eval()
self.diffusion.eval()
self.clvp.eval()
self.vocoder.eval()
def train_step(self):
raise NotImplementedError("Tortoise Training is not implemented")