How to extract pitch from an English speaking audio file?

[abusaadp]

Hi,
I would like to know how to extract pitch from an English speaking audio file using this pitch extractor in this repository.

[Respaired]

Hi.

def load_F0_models(path):

    F0_model = JDCNet(num_class=1, seq_len=192)
    params = torch.load(path, map_location='cpu')
    F0_model.load_state_dict(params)
    _ = F0_model.eval()
    
    return F0_model.to('cuda')

F0_path = "path_to_your_model"
pitch_extractor = load_F0_models(F0_path)



to_mel = torchaudio.transforms.MelSpectrogram(
    n_mels=80, n_fft=2048, win_length=1200, hop_length=300)
mean, std = -4, 4

def preprocess(wave):
    wave_tensor = torch.from_numpy(wave).float()
    mel_tensor = to_mel(wave_tensor)
    mel_tensor = (torch.log(1e-5 + mel_tensor.unsqueeze(0)) - mean) / std
    return mel_tensor


audio = librosa.load(your audio file here, sr=24_000)[0]


@torch.no_grad()
def extract_f0(audio):
    mel = preprocess(audio)
    f0, _, _ = pitch_extractor(mel)
    mean_f0 = f0.mean()
    std_f0 = f0.std()
    return float(f"{mean_f0:.4f}"), float(f"{std_f0:.4f}")

extract_f0(audio)

[abusaadp]

In the below function

def load_F0_models(path):

    F0_model = JDCNet(num_class=1, seq_len=192)
    params = torch.load(path, map_location='cpu')
    F0_model.load_state_dict(params)
    _ = F0_model.eval()

    return F0_model.to(cpu)

I got this error

RuntimeError: Error(s) in loading state_dict for JDCNet: Missing key(s) in state_dict: "conv_block.0.weight", "conv_block.1.weight", "conv_block.1.bias", "conv_block.1.running_mean", "conv_block.1.running_var", "conv_block.3.weight", "res_block1.pre_conv.0.weight", "res_block1.pre_conv.0.bias", "res_block1.pre_conv.0.running_mean", "res_block1.pre_conv.0.running_var", "res_block1.conv.0.weight", "res_block1.conv.1.weight", "res_block1.conv.1.bias", "res_block1.conv.1.running_mean", "res_block1.conv.1.running_var", "res_block1.conv.3.weight", "res_block1.conv1by1.weight", "res_block2.pre_conv.0.weight", "res_block2.pre_conv.0.bias", "res_block2.pre_conv.0.running_mean", "res_block2.pre_conv.0.running_var", "res_block2.conv.0.weight", "res_block2.conv.1.weight", "res_block2.conv.1.bias", "res_block2.conv.1.running_mean", "res_block2.conv.1.running_var", "res_block2.conv.3.weight", "res_block2.conv1by1.weight", "res_block3.pre_conv.0.weight", "res_block3.pre_conv.0.bias", "res_block3.pre_conv.0.running_mean", "res_block3.pre_conv.0.running_var", "res_block3.conv.0.weight", "res_block3.conv.1.weight", "res_block3.conv.1.bias", "res_block3.conv.1.running_mean", "res_block3.conv.1.running_var", "res_block3.conv.3.weight", "res_block3.conv1by1.weight", "pool_block.0.weight", "pool_block.0.bias", "pool_block.0.running_mean", "pool_block.0.running_var", "detector_conv.0.weight", "detector_conv.1.weight", "detector_conv.1.bias", "detector_conv.1.running_mean", "detector_conv.1.running_var", "bilstm_classifier.weight_ih_l0", "bilstm_classifier.weight_hh_l0", "bilstm_classifier.bias_ih_l0", "bilstm_classifier.bias_hh_l0", "bilstm_classifier.weight_ih_l0_reverse", "bilstm_classifier.weight_hh_l0_reverse", "bilstm_classifier.bias_ih_l0_reverse", "bilstm_classifier.bias_hh_l0_reverse", "bilstm_detector.weight_ih_l0", "bilstm_detector.weight_hh_l0", "bilstm_detector.bias_ih_l0", "bilstm_detector.bias_hh_l0", "bilstm_detector.weight_ih_l0_reverse", "bilstm_detector.weight_hh_l0_reverse", "bilstm_detector.bias_ih_l0_reverse", "bilstm_detector.bias_hh_l0_reverse", "classifier.weight", "classifier.bias", "detector.weight", "detector.bias". Unexpected key(s) in state_dict: "optimizer", "scheduler", "steps", "epochs", "model".

So I changed the function to the following

def load_F0_models(path):

    F0_model = JDCNet(num_class=1, seq_len=192)
    params = torch.load(path, map_location='cpu')
    F0_model.load_state_dict(params['model'])
    _ = F0_model.eval()

    return F0_model.to('cpu')

Then I got this error

ValueError: expected 4D input (got 3D input)

To fix this I made the following change

mel_tensor = (torch.log(1e-5 + mel_tensor.unsqueeze(0).unsqueeze(0)) - mean) / std

Now I am getting this error

RuntimeError: Sizes of tensors must match except in dimension 1. Expected size 80 but got size 100 for tensor number 3 in the list.

How to fix it?

P.S : Hope the corrections I made are correct

[Respaired]

load the JDCnet class from one of the StyleTTS repos, not the one provided here.

RuntimeError: Sizes of tensors must match except in dimension 1. Expected size 80 but got size 100 for tensor number 3 in the list.

I guess the mel you're feeding it has 100bins? the default settings in this repo (and all other related repos) are limited to 80 bins. you must change them accordingly, which means changing the dims in Maxpool and the detector iirc.

[abusaadp]

If I use the bst.t7 model from StyleTTS2 repo and F0_model.load_state_dict(params["net"]) then I get the same error

RuntimeError: Sizes of tensors must match except in dimension 1. Expected size 80 but got size 100 for tensor number 3 in the list.

[Respaired]

what is the config you're using though? please show me your mel preprocessing steps

[abuspeak]

I am using this config from StyleTTS2

log_dir: "Models/LJSpeech"
first_stage_path: "first_stage.pth"
save_freq: 2
log_interval: 10
device: "cuda"
epochs_1st: 200 # number of epochs for first stage training (pre-training)
epochs_2nd: 100 # number of peochs for second stage training (joint training)
batch_size: 16
max_len: 400 # maximum number of frames
pretrained_model: ""
second_stage_load_pretrained: true # set to true if the pre-trained model is for 2nd stage
load_only_params: false # set to true if do not want to load epoch numbers and optimizer parameters

F0_path: "Utils/JDC/bst.t7"
ASR_config: "Utils/ASR/config.yml"
ASR_path: "Utils/ASR/epoch_00080.pth"
PLBERT_dir: 'Utils/PLBERT/'

data_params:
  train_data: "Data/train_list.txt"
  val_data: "Data/val_list.txt"
  root_path: "/local/LJSpeech-1.1/wavs"
  OOD_data: "Data/OOD_texts.txt"
  min_length: 50 # sample until texts with this size are obtained for OOD texts

preprocess_params:
  sr: 24000
  spect_params:
    n_fft: 2048
    win_length: 1200
    hop_length: 300

model_params:
  multispeaker: false

  dim_in: 64 
  hidden_dim: 512
  max_conv_dim: 512
  n_layer: 3
  n_mels: 80

  n_token: 178 # number of phoneme tokens
  max_dur: 50 # maximum duration of a single phoneme
  style_dim: 128 # style vector size
  
  dropout: 0.2

  # config for decoder
  decoder: 
      type: 'istftnet' # either hifigan or istftnet
      resblock_kernel_sizes: [3,7,11]
      upsample_rates :  [10, 6]
      upsample_initial_channel: 512
      resblock_dilation_sizes: [[1,3,5], [1,3,5], [1,3,5]]
      upsample_kernel_sizes: [20, 12]
      gen_istft_n_fft: 20
      gen_istft_hop_size: 5
      
  # speech language model config
  slm:
      model: 'microsoft/wavlm-base-plus'
      sr: 16000 # sampling rate of SLM
      hidden: 768 # hidden size of SLM
      nlayers: 13 # number of layers of SLM
      initial_channel: 64 # initial channels of SLM discriminator head
  
  # style diffusion model config
  diffusion:
    embedding_mask_proba: 0.1
    # transformer config
    transformer:
      num_layers: 3
      num_heads: 8
      head_features: 64
      multiplier: 2

    # diffusion distribution config
    dist:
      sigma_data: 0.2 # placeholder for estimate_sigma_data set to false
      estimate_sigma_data: true # estimate sigma_data from the current batch if set to true
      mean: -3.0
      std: 1.0
  
loss_params:
    lambda_mel: 5. # mel reconstruction loss
    lambda_gen: 1. # generator loss
    lambda_slm: 1. # slm feature matching loss
    
    lambda_mono: 1. # monotonic alignment loss (1st stage, TMA)
    lambda_s2s: 1. # sequence-to-sequence loss (1st stage, TMA)
    TMA_epoch: 50 # TMA starting epoch (1st stage)

    lambda_F0: 1. # F0 reconstruction loss (2nd stage)
    lambda_norm: 1. # norm reconstruction loss (2nd stage)
    lambda_dur: 1. # duration loss (2nd stage)
    lambda_ce: 20. # duration predictor probability output CE loss (2nd stage)
    lambda_sty: 1. # style reconstruction loss (2nd stage)
    lambda_diff: 1. # score matching loss (2nd stage)
    
    diff_epoch: 20 # style diffusion starting epoch (2nd stage)
    joint_epoch: 50 # joint training starting epoch (2nd stage)

optimizer_params:
  lr: 0.0001 # general learning rate
  bert_lr: 0.00001 # learning rate for PLBERT
  ft_lr: 0.00001 # learning rate for acoustic modules
  
slmadv_params:
  min_len: 400 # minimum length of samples
  max_len: 500 # maximum length of samples
  batch_percentage: 0.5 # to prevent out of memory, only use half of the original batch size
  iter: 10 # update the discriminator every this iterations of generator update
  thresh: 5 # gradient norm above which the gradient is scaled
  scale: 0.01 # gradient scaling factor for predictors from SLM discriminators
  sig: 1.5 # sigma for differentiable duration modeling