""" This file manages the experiments, see the main function for changing the settings """ import os import random import time import pandas as pd from gtts import gTTS from keras_preprocessing.text import Tokenizer def main(): """ This function runs the process of the experiments. Iterates over the parameters and output the results. :return: Nothing """ # Some settings for the files we will use saved_file_type = 'pkl' midi_pickle = os.path.join(PICKLES_FOLDER, f"midi.{saved_file_type}") midi_folder = os.path.join(DATA_PATH, "midi_files") # Read a pre-trained word2vec dictionary word2vec_path = os.path.join(PICKLES_FOLDER, f"{WORD2VEC_FILENAME}.{saved_file_type}") pre_trained = os.path.join(INPUT_FOLDER, f"{GLOVE_FILE_NAME}.txt") # Get the embedding dictionary that maps between word to a vector word2vec = get_word2vec(word2vec_path=word2vec_path, pre_trained=pre_trained, vector_size=VECTOR_SIZE, encoding=ENCODING) # load the training and testing set that provided by the course staff train_pickle_path = os.path.join(PICKLES_FOLDER, f'{TRAIN_NAME}.{saved_file_type}') input_train_path = os.path.join(INPUT_FOLDER, INPUT_TRAINING_SET) training_set = get_input_sets(input_file=input_train_path, pickle_path=train_pickle_path, word2vec=word2vec, midi_folder=midi_folder) test_pickle_path = os.path.join(PICKLES_FOLDER, f'{TEST_NAME}.{saved_file_type}') input_test_path = os.path.join(INPUT_FOLDER, INPUT_TESTING_SET) testing_set = get_input_sets(input_file=input_test_path, pickle_path=test_pickle_path, word2vec=word2vec, midi_folder=midi_folder) artists = training_set['artists'] + testing_set['artists'] songs_names = training_set['names'] + testing_set['names'] lyrics = training_set['lyrics'] + testing_set['lyrics'] tokenizer = Tokenizer() tokenizer.fit_on_texts(lyrics) total_words = len(tokenizer.word_index) + 1 encoded_lyrics_list = tokenizer.texts_to_sequences(lyrics) index2word = tokenizer.index_word melodies = get_midi_files(midi_folder=midi_folder, midi_pickle=midi_pickle, artists=artists, names=songs_names) train_encoded_lyrics_list = encoded_lyrics_list[:len(training_set['lyrics'])] test_encoded_lyrics_list = encoded_lyrics_list[len(training_set['lyrics']):] melody_pickle = os.path.join(PICKLES_FOLDER, "melody_data." + saved_file_type) comb_dict = {'seed': [], 'seq_length': [], 'learning_rate': [], 'batch_size': [], 'epochs': [], 'patience': [], 'min_delta': [], 'melody_method': [], 'model_names': [], 'cos_sim_1_gram': [], 'cos_sim_2_gram': [], 'cos_sim_3_gram': [], 'cos_sim_5_gram': [], 'cos_sim_max_gram': [], 'polarity_diff': [], 'subjectivity_diff': [], 'loss_val': [], 'accuracy': []} word2vec_matrix = get_word2vec_matrix(total_words=total_words, index2word=index2word, word2vec=word2vec, vector_size=VECTOR_SIZE) for seed in seeds_list: for sl in seq_length_list: sets_dict = create_sets( train_encoded_lyrics_list=train_encoded_lyrics_list, test_encoded_lyrics_list=test_encoded_lyrics_list, total_words=total_words, seq_length=sl, validation_set_size=VALIDATION_SET_SIZE, seed=seed) training_sequences = sets_dict['train'][1].shape[0] + sets_dict['validation'][1].shape[0] for melody_method in melody_extraction: m_train, m_val, m_test = get_melody_data_sets( train_num=training_sequences, val_size=VALIDATION_SET_SIZE, melodies_list=melodies, sequence_length=sl, encoded_lyrics_matrix=encoded_lyrics_list, pkl_file_path=melody_pickle, seed=seed, feature_method=melody_method) melody_feature_vector_size = m_train.shape[2] for l in learning_rate_list: for bs in batch_size_list: for ep in epochs_list: for pa in patience_list: for md in min_delta_list: for u in units_list: for m_name in model_names_list: run_combination(comb_dict, sl, bs, ep, index2word, l, md, pa, seed, testing_set['artists'], melody_method, testing_set['lyrics'], testing_set['names'], total_words, u, word2vec, word2vec_matrix, tokenizer, sets_dict['train'][0], sets_dict['validation'][0], sets_dict['test'][0], m_train, m_val, m_test, sets_dict['train'][1], sets_dict['validation'][1], sets_dict['test'][1], m_name, melody_feature_vector_size) if m_name == 'lyrics': break # Here we save all the results to a csv file comb_df = pd.DataFrame.from_dict(comb_dict) comb_df.to_csv(COMB_PATH, index=False) def run_combination(comb_dict, seq_length, batch_size, epochs, index2word, learning_rate, min_delta, patience, seed, test_artists, melody_extraction_method, test_lyrics, test_names, total_words, units, word2vec, word2vec_matrix, tokenizer, x_train, x_val, x_test, m_train, m_val, m_test, y_train, y_val, y_test, model_name, melody_num_features): """ This function runs a combination with a specific settings and training or testing set :param melody_extraction_method: The method used to extract melody features (naive or with meta data) :param comb_dict: dictionary of all the results :param seq_length: this is the input sequence length we used for the LSTM model :param batch_size: the batch size for the model :param epochs: number of epochs for the model :param index2word: a dictionary maps between index and words. :param learning_rate: learning rate for the model :param min_delta: minimum delta for early stopping of the model :param patience: patience fo the early stopping of the model :param seed: for the random state :param test_artists: list of artist in the training set :param test_lyrics: list of lyrics in the training set :param test_names: list of songs name in the training set :param total_words: total size of the vocabulary :param units: number of LSTM units :param word2vec: dictionary maps between a word and a vector :param word2vec_matrix: a matrix of words (rows) and vectors (columns) of the word2vec :param tokenizer: Tokenizer object :param x_train: lyrics training set :param x_val: lyrics validation set :param x_test: lyrics testing xet :param m_train: melody training set :param m_val: melody validation set :param m_test: melody testing set :param y_train: training output words :param y_val: validation output words :param y_test: testing output words :param model_name: the name of the model we want to use in this function :param melody_num_features: size of the melody vector :return: Nothing """ model_save_type = 'h5' # file type initialize_seed(seed) # files paths parameters_name = f'seq_lens_{seq_length}_seed_{seed}_u_{units}_lr_{learning_rate}_bs_{batch_size}_ep_{epochs}_' \ f'val_{VALIDATION_SET_SIZE}_pa_{patience}_md_{min_delta}_mn_{model_name}' if not model_name == 'lyrics': parameters_name += f'_fm_{melody_extraction_method}' # A path for the weights load_weights_path = os.path.join(WEIGHTS_FOLDER, f'weights_{parameters_name}.{model_save_type}') model = None if model_name == 'lyrics': model = LSTMLyrics(seed=seed, loss=LOSS, metrics=METRICS, optimizer=OPTIMIZER, learning_rate=learning_rate, total_words=total_words, seq_length=seq_length, vector_size=VECTOR_SIZE, word2vec_matrix=word2vec_matrix, units=units) elif model_name == 'melodies_lyrics': x_train = [x_train, m_train] x_val = [x_val, m_val] x_test = [x_test, m_test] model = LSTMLyricsMelodies(seed=seed, loss=LOSS, metrics=METRICS, optimizer=OPTIMIZER, learning_rate=learning_rate, total_words=total_words, seq_length=seq_length, vector_size=VECTOR_SIZE, word2vec_matrix=word2vec_matrix, units=units, melody_num_features=melody_num_features) model.fit(weights_file=load_weights_path, batch_size=batch_size, epochs=epochs, patience=patience, min_delta=min_delta, x_train=x_train, y_train=y_train, x_val=x_val, y_val=y_val) loss_val, accuracy = model.evaluate(x_test=x_test, y_test=y_test, batch_size=batch_size) print(f'Loss on Testing set: {loss_val}') print(f'Accuracy on Testing set: {accuracy}') all_original_lyrics, all_generated_lyrics = generate_lyrics( model_name=model_name, word_index=index2word, seq_length=seq_length, model=model, tokenizer=tokenizer, artists=test_artists, lyrics=test_lyrics, names=test_names, word2vec=word2vec, melodies=m_test ) cos_sim_1_gram = calculate_cosine_similarity_n_gram(all_generated_lyrics=all_generated_lyrics, all_original_lyrics=all_original_lyrics, n=1, word2vec=word2vec) print(f'Mean Cosine Similarity (1-gram): {cos_sim_1_gram}') cos_sim_2_gram = calculate_cosine_similarity_n_gram(all_generated_lyrics=all_generated_lyrics, all_original_lyrics=all_original_lyrics, n=2, word2vec=word2vec) print(f'Mean Cosine Similarity (2-gram): {cos_sim_2_gram}') cos_sim_3_gram = calculate_cosine_similarity_n_gram(all_generated_lyrics=all_generated_lyrics, all_original_lyrics=all_original_lyrics, n=3, word2vec=word2vec) print(f'Mean Cosine Similarity (3-gram): {cos_sim_3_gram}') cos_sim_5_gram = calculate_cosine_similarity_n_gram(all_generated_lyrics=all_generated_lyrics, all_original_lyrics=all_original_lyrics, n=5, word2vec=word2vec) print(f'Mean Cosine Similarity (5-gram): {cos_sim_5_gram}') cos_sim = calculate_cosine_similarity(all_generated_lyrics=all_generated_lyrics, all_original_lyrics=all_original_lyrics, word2vec=word2vec) print(f'Mean Cosine Similarity (Max-gram): {cos_sim}') pol_dif = get_polarity_diff(all_generated_lyrics=all_generated_lyrics, all_original_lyrics=all_original_lyrics) print(f'Mean Polarity Difference: {pol_dif}') subj_dif = get_subjectivity_diff(all_generated_lyrics=all_generated_lyrics, all_original_lyrics=all_original_lyrics) print(f'Mean Subjectivity Difference: {subj_dif}') update_comb_dict(batch_size, comb_dict, cos_sim, cos_sim_1_gram, cos_sim_2_gram, cos_sim_3_gram, cos_sim_5_gram, epochs, learning_rate, min_delta, model_name, patience, pol_dif, seed, seq_length, subj_dif, melody_extraction_method, loss_val, accuracy) def update_comb_dict(batch_size, comb_dict, cos_sim, cos_sim_1_gram, cos_sim_2_gram, cos_sim_3_gram, cos_sim_5_gram, epochs, learning_rate, min_delta, model_name, patience, pol_dif, seed, seq_length, subj_dif, melody_extraction_method, loss_val, accuracy): """ This function update the combination dictionary to write to csv :param accuracy: accuracy on the testing set :param loss_val: loss on the testing set :param batch_size: the batch size for the model :param comb_dict: the results dictionary :param cos_sim: the similarity score between the original and the generated sentence :param cos_sim_1_gram: the similarity score between each 1 gram of original and the generated sentence :param cos_sim_2_gram: the similarity score between each 2 gram of original and the generated sentence :param cos_sim_3_gram: the similarity score between each 3 gram of original and the generated sentence :param cos_sim_5_gram: the similarity score between each 5 gram of original and the generated sentence :param epochs: number of epochs for the model :param learning_rate: learning rate for the model :param min_delta: minimum delta for early stopping of the model :param model_name: The model name we want to test :param patience: patience fo the early stopping of the model :param pol_dif: the difference polarity score between the original and the generated sentence :param seed: for the random state :param seq_length: length of the given sequences :param subj_dif: the difference subjective score between the original and the generated sentence :param melody_extraction_method: The method used to extract melody features (naive or with meta data) :return: Nothing """ comb_dict['seed'].append(seed) comb_dict['seq_length'].append(seq_length) comb_dict['learning_rate'].append(learning_rate) comb_dict['batch_size'].append(batch_size) comb_dict['epochs'].append(epochs) comb_dict['patience'].append(patience) comb_dict['min_delta'].append(min_delta) comb_dict['model_names'].append(model_name) comb_dict['cos_sim_1_gram'].append(cos_sim_1_gram) comb_dict['cos_sim_2_gram'].append(cos_sim_2_gram) comb_dict['cos_sim_3_gram'].append(cos_sim_3_gram) comb_dict['cos_sim_5_gram'].append(cos_sim_5_gram) comb_dict['cos_sim_max_gram'].append(cos_sim) comb_dict['polarity_diff'].append(pol_dif) comb_dict['subjectivity_diff'].append(subj_dif) comb_dict['melody_method'].append(melody_extraction_method) comb_dict['loss_val'].append(loss_val) comb_dict['accuracy'].append(accuracy) def generate_song_given_sequence(model_name, model, tokenizer, seed_words, vector_of_indices, required_length, artist, name, index_value, melodies_song): """ This function generates a new song :param model_name: model name :param melodies_song: a matrix contains the melodies of this song :param model: :param tokenizer: :param seed_words: :param vector_of_indices: :param required_length: :param artist: :param name: :param index_value: :return: Nothing """ new_song_lyrics: list = [seed_words] for word_i in range(required_length): if model_name == 'lyrics': # Different input for lyrics alone and lyrics and melodies. voc_prob = model.predict(vector_of_indices) else: melody_seq = np.expand_dims(a=melodies_song[word_i], axis=0) voc_prob = model.predict([vector_of_indices, melody_seq]) voc_prob = voc_prob.T # Transpose the array word_index_array = np.arange(voc_prob.size) # This line select a word based on the predicted probabilities index_of_selected_word = random.choices(word_index_array, k=1, weights=voc_prob) selected_word = find_word_by_index(word_index=index_of_selected_word[0], tokenizer=tokenizer) index_of_selected_word_array = np.array(np.array(index_of_selected_word).reshape(1, 1)) vector_of_indices = np.append(vector_of_indices, index_of_selected_word_array, axis=1) remove_index = 0 vector_of_indices = np.delete(vector_of_indices, remove_index, 1) new_song_lyrics.append(selected_word) final_text = ' '.join(new_song_lyrics) if WRITE_TO_MP3: lyrics_to_mp3 = gTTS(text=final_text, lang='en', slow=False) lyrics_to_mp3.save(os.path.join(OUTPUT_FOLDER, f"{artist}_{name}_{index_value}.mp3")) return final_text def find_word_by_index(word_index, tokenizer): """ This function returns the word given the index :param word_index: the index of the word we want to find :param tokenizer: object :return: the word at that index """ for word, index in tokenizer.word_index.items(): if index == word_index: return word def generate_lyrics(model_name, word_index, seq_length, model, tokenizer, artists, lyrics, names, word2vec, melodies) -> (list, list): """ This function creates lyrics for each song in the testing set :param melodies: a 3D array that maps sequence and the melodies features (2D array (sequence size, melody vector)). :param model_name: The model name we want to test :param word_index: A dictionary maps between index to word :param seq_length: length of the given sequences :param model: the learned model :param tokenizer: the tokenizer object :param artists: list of artists in the testing set :param lyrics: list of lyrics in the testing set :param names: list of song names in the testing set :param word2vec: A dictionary maps between word to embedding vector :return: lists of original and generated songs and """ all_original_lyrics = [] all_generated_lyrics = [] start_index_melody = 0 for artist, name, lyrics in zip(artists, names, lyrics): print('-' * 100) print(f'Original lyrics for {artist} - {name} are: "{lyrics}"') relevant_words_in_song = [] find_relevant_words(lyrics, relevant_words_in_song, word2vec) number_of_seq = len(relevant_words_in_song) - seq_length + 1 end_index_melody = start_index_melody + number_of_seq melodies_song = melodies[start_index_melody:end_index_melody, :, :] required_length = len(relevant_words_in_song) - (seq_length * TESTING_SEED_TEXT_PER_SONG) for seed_index in range(TESTING_SEED_TEXT_PER_SONG): # We select three different word\sentence as seed for the new song starting_index = 0 + seed_index * seq_length ending_index = starting_index + seq_length song_first_word_in_word2vec = relevant_words_in_song[starting_index:ending_index] song_first_indices = [] for word in song_first_word_in_word2vec: word_i = [k for k, v in word_index.items() if v == word][0] song_first_indices.append(word_i) encoded_test = np.asarray(song_first_indices).reshape((1, seq_length)) seed_text = ' '.join(song_first_word_in_word2vec) generated_text = generate_song_given_sequence(model_name, model, tokenizer, seed_text, encoded_test, required_length, artist, name, seed_index, melodies_song) gen_list = generated_text.split(' ') all_generated_lyrics.append(gen_list.copy()[seq_length:]) original_starting_index = starting_index + seq_length original_ending_index = original_starting_index + required_length original_lyrics = relevant_words_in_song[original_starting_index:original_ending_index] all_original_lyrics.append(original_lyrics) gen_list.insert(seq_length, '\n') generated_text = ' '.join(gen_list) print(f'Seed text: {generated_text}, required {required_length} words') print('-' * 100) start_index_melody = end_index_melody + 1 return all_original_lyrics, all_generated_lyrics def find_relevant_words(lyrics, selected_words, word2vec): """ This loop selects all the relevant words in the pre-defined word2vec :param lyrics: :param selected_words: :param word2vec: :return: """ for word in lyrics.split(): if word in word2vec and word not in selected_words: selected_words.append(word) def initialize_seed(seed): """ Initialize all relevant environments with the seed. """ os.environ['PYTHONHASHSEED'] = str(seed) random.seed(seed) np.random.seed(seed) def folder_exists(path): """ This function checks if folder path is exists, in case not, the function creates the folder. :param path: folder path """ if not os.path.exists(path): os.mkdir(path) if __name__ == '__main__': # Environment settings IS_COLAB = (os.name == 'posix') LOAD_DATA = not (os.name == 'posix') path_separator = os.path.sep IS_EXPERIMENT = False WRITE_TO_MP3 = False if IS_COLAB: # the google drive folder we used DATA_PATH = os.path.sep + os.path.join('content', 'drive', 'My\ Drive', 'datasets', 'midi').replace('\\', '') IS_EXPERIMENT = True else: # locally from data_loader import get_word2vec from data_loader import get_input_sets from data_loader import get_midi_files from lstm_lyrics import LSTMLyrics from lstm_melodies_lyrics import LSTMLyricsMelodies from prepare_data import get_word2vec_matrix from prepare_data import create_sets from compute_score import calculate_cosine_similarity from compute_score import get_polarity_diff from compute_score import get_subjectivity_diff from compute_score import calculate_cosine_similarity_n_gram from extract_melodies_features import * DATA_PATH = os.path.join('.\\', 'midi') os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # PATHS TRAIN_NAME = 'train' INPUT_TRAINING_SET = f"lyrics_{TRAIN_NAME}_set.csv" TEST_NAME = 'test' INPUT_TESTING_SET = f"lyrics_{TEST_NAME}_set.csv" OUTPUT_FOLDER = os.path.join(DATA_PATH, 'output_files') folder_exists(OUTPUT_FOLDER) INPUT_FOLDER = os.path.join(DATA_PATH, 'input_files') folder_exists(INPUT_FOLDER) PICKLES_FOLDER = os.path.join(DATA_PATH, 'pickles') folder_exists(PICKLES_FOLDER) WEIGHTS_FOLDER = os.path.join(DATA_PATH, 'weights') folder_exists(WEIGHTS_FOLDER) WORD2VEC_FILENAME = 'word2vec' RESULTS_FILE_NAME = 'results.csv' COMB_PATH = os.path.join(OUTPUT_FOLDER, RESULTS_FILE_NAME) GLOVE_FILE_NAME = 'glove.6B.300d' ENCODING = 'utf-8' LOSS = 'categorical_crossentropy' METRICS = ['accuracy'] VECTOR_SIZE = 300 VALIDATION_SET_SIZE = 0.2 TESTING_SEED_TEXT_PER_SONG = 3 OPTIMIZER = 'adam' if IS_EXPERIMENT: # Experiments settings seeds_list = [0] learning_rate_list = [0.01] batch_size_list = [32, 64] epochs_list = [10] patience_list = [0] min_delta_list = [0.1] units_list = [256] seq_length_list = [1, 5, 20] model_names_list = ['melodies_lyrics', 'lyrics'] melody_extraction = ['naive'] # melody_extraction = ['naive', 'with_meta_features'] else: # Final settings seeds_list = [0] learning_rate_list = [0.01] batch_size_list = [32] epochs_list = [10] patience_list = [0] min_delta_list = [0.1] units_list = [256] seq_length_list = [1] model_names_list = ['melodies_lyrics'] melody_extraction = ['naive'] # model_names_list = ['melodies_lyrics', 'lyrics'] # melody_extraction = ['naive', 'with_meta_features'] start_time = time.time() main() print("--- %s seconds ---" % (time.time() - start_time))