From 18a65ba587eea362c20bc1fe9a4f27cc7822ba3e Mon Sep 17 00:00:00 2001 From: kelly anne Date: Wed, 5 Jun 2024 13:02:27 +0200 Subject: [PATCH 01/14] add flow to get prediction --- README.md | 14 ++++++++++++++ koregraph/cli/generate_features.py | 9 +++++++++ koregraph/params.py | 6 ++++++ koregraph/tools/video_builder.py | 6 +----- pyproject.toml | 3 ++- 5 files changed, 32 insertions(+), 6 deletions(-) create mode 100644 koregraph/cli/generate_features.py diff --git a/README.md b/README.md index 97ac8e6..f081fea 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,20 @@ poetry install poetry install --with dev,viewer ``` +## Get a prediction +**Generate features and output pkl files** +```bash +poetry run generate +``` +**Train a model** +```bash +poetry run train -m model_kelly +``` +**Get a video** +```bash +poetry run predict -m model_kelly -a mBR0 +``` + ## Scripts **Export presentation (github pages)** diff --git a/koregraph/cli/generate_features.py b/koregraph/cli/generate_features.py new file mode 100644 index 0000000..5b22306 --- /dev/null +++ b/koregraph/cli/generate_features.py @@ -0,0 +1,9 @@ +from koregraph.api.machine_learning.pickle_creation import generate_pickle_files + + +def main(): + generate_pickle_files() + + +if __name__ == "__main__": + main() diff --git a/koregraph/params.py b/koregraph/params.py index 9a94053..25693a8 100644 --- a/koregraph/params.py +++ b/koregraph/params.py @@ -54,6 +54,12 @@ IMAGE_DIRECTORY.mkdir(parents=True, exist_ok=True) print(f"Created directory'{IMAGE_DIRECTORY}'") +KEYPOINTS_BUILDER_TEMP_DIRECTORY = Path( + environ.get("KEYPOINTS_BUILDER_TEMP_DIRECTORY", "generated/videos") +) +KEYPOINTS_BUILDER_TEMP_DIRECTORY.mkdir(parents=True, exist_ok=True) + + WEIGHTS_BACKUP_DIRECTORY: Path = Path( environ.get("WEIGHTS_BACKUP_DIRECTORY", "temp/backup") ) diff --git a/koregraph/tools/video_builder.py b/koregraph/tools/video_builder.py index 6c9ef04..6e659b2 100644 --- a/koregraph/tools/video_builder.py +++ b/koregraph/tools/video_builder.py @@ -11,13 +11,9 @@ from koregraph.models.aist_file import AISTFile from koregraph.models.choregraphy import Choregraphy from koregraph.models.posture import Posture +from koregraph.params import KEYPOINTS_BUILDER_TEMP_DIRECTORY -KEYPOINTS_BUILDER_TEMP_DIRECTORY = Path( - environ.get("KEYPOINTS_BUILDER_TEMP_DIRECTORY", "temp") -) -KEYPOINTS_BUILDER_TEMP_DIRECTORY.mkdir(parents=True, exist_ok=True) - COLORS = [ [255, 0, 0], [255, 85, 0], diff --git a/pyproject.toml b/pyproject.toml index 0f7510d..6301817 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,8 +60,9 @@ tensorflow = "^2.16.1" [tool.poetry.scripts] +generate = "koregraph.cli.generate_features:main" viewer = "koregraph.cli.viewer:main" -chunk = "koregraph.cli.chunk:main" +chunk = "koregraph.cli.chunk_command:main" train = "koregraph.cli.model:main" predict = "koregraph.cli.predict:main" From 995c45521caaa1889767d683d0a00df899d83ad4 Mon Sep 17 00:00:00 2001 From: kelly anne Date: Wed, 5 Jun 2024 16:15:11 +0200 Subject: [PATCH 02/14] read 10 chore --- .../api/machine_learning/load_dataset.py | 21 ++++++++++++++++++- koregraph/api/posture_proc.py | 5 +++-- koregraph/params.py | 14 +++++++------ koregraph/tools/video_builder.py | 4 ++-- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/koregraph/api/machine_learning/load_dataset.py b/koregraph/api/machine_learning/load_dataset.py index 6031238..7966932 100644 --- a/koregraph/api/machine_learning/load_dataset.py +++ b/koregraph/api/machine_learning/load_dataset.py @@ -4,7 +4,7 @@ from typing import Any -from numpy import ndarray +from numpy import ndarray, append from koregraph.utils.pickle import load_pickle_object from koregraph.params import GENERATED_PICKLE_DIRECTORY @@ -21,6 +21,25 @@ def load_preprocess_dataset() -> tuple[ndarray, ndarray]: GENERATED_PICKLE_DIRECTORY / "generated_gBR_sFM_cAll_d04_mBR0_ch01.pkl" ) + files = [ + "generated_gBR_sFM_cAll_d04_mBR1_ch02.pkl", + "generated_gBR_sFM_cAll_d04_mBR2_ch03.pkl", + "generated_gBR_sFM_cAll_d04_mBR3_ch04.pkl", + "generated_gBR_sFM_cAll_d04_mBR4_ch05.pkl", + "generated_gBR_sFM_cAll_d04_mBR4_ch07.pkl", + "generated_gBR_sFM_cAll_d04_mBR5_ch06.pkl", + "generated_gBR_sFM_cAll_d05_mBR2_ch09.pkl", + "generated_gBR_sFM_cAll_d05_mBR3_ch10.pkl", + "generated_gBR_sFM_cAll_d05_mBR4_ch11.pkl", + "generated_gBR_sFM_cAll_d05_mBR4_ch13.pkl", + "generated_gKR_sFM_cAll_d28_mKR0_ch01.pkl", + ] + + for file in files: + y_tmp, X_tmp = load_pickle_object(GENERATED_PICKLE_DIRECTORY / file) + X = append(X, X_tmp, axis=0) + y = append(y, y_tmp, axis=0) + return X, y diff --git a/koregraph/api/posture_proc.py b/koregraph/api/posture_proc.py index 8d6befe..23676ad 100644 --- a/koregraph/api/posture_proc.py +++ b/koregraph/api/posture_proc.py @@ -1,7 +1,7 @@ -from numpy import array, ndarray +from numpy import array, ndarray, nan_to_num from koregraph.params import KEYPOINTS_DIRECTORY, FRAME_FORMAT from pickle import load as load_pickle - +from numpy import isnan from typing import Tuple @@ -19,6 +19,7 @@ def generate_posture_array( with open(KEYPOINTS_DIRECTORY / (choregraphy_name), "rb") as f: data = load_pickle(f) postures = data["keypoints2d"][0, :, :, :2] + postures = nan_to_num(postures, 0) postures[:, :, 0] = postures[:, :, 0] / frame_format[0] postures[:, :, 1] = postures[:, :, 1] / frame_format[1] diff --git a/koregraph/params.py b/koregraph/params.py index 25693a8..201fb5b 100644 --- a/koregraph/params.py +++ b/koregraph/params.py @@ -54,29 +54,31 @@ IMAGE_DIRECTORY.mkdir(parents=True, exist_ok=True) print(f"Created directory'{IMAGE_DIRECTORY}'") -KEYPOINTS_BUILDER_TEMP_DIRECTORY = Path( +KEYPOINTS_BUILDER_TEMP_DIRECTORY = PROJECT_ROOT.joinpath( environ.get("KEYPOINTS_BUILDER_TEMP_DIRECTORY", "generated/videos") ) KEYPOINTS_BUILDER_TEMP_DIRECTORY.mkdir(parents=True, exist_ok=True) -WEIGHTS_BACKUP_DIRECTORY: Path = Path( +WEIGHTS_BACKUP_DIRECTORY: Path = PROJECT_ROOT.joinpath( environ.get("WEIGHTS_BACKUP_DIRECTORY", "temp/backup") ) WEIGHTS_BACKUP_DIRECTORY.mkdir(parents=True, exist_ok=True) -MODEL_OUTPUT_DIRECTORY: Path = Path(environ.get("OUT_DIRECTORY", "generated/models")) +MODEL_OUTPUT_DIRECTORY: Path = PROJECT_ROOT.joinpath( + environ.get("OUT_DIRECTORY", "generated/models") +) MODEL_OUTPUT_DIRECTORY.mkdir(parents=True, exist_ok=True) -PREDICTION_OUTPUT_DIRECTORY: Path = Path( +PREDICTION_OUTPUT_DIRECTORY: Path = PROJECT_ROOT.joinpath( environ.get("PREDICTION_OUTPUT_DIRECTORY", "generated/predictions/") ) PREDICTION_OUTPUT_DIRECTORY.mkdir(parents=True, exist_ok=True) -GENERATED_PICKLE_DIRECTORY: Path = Path( +GENERATED_PICKLE_DIRECTORY: Path = PROJECT_ROOT.joinpath( environ.get( "GENERATED_PICKLE_DIRECTORY", - "generated/data/pickles", + "generated/inputs", ) ) diff --git a/koregraph/tools/video_builder.py b/koregraph/tools/video_builder.py index 6e659b2..021c065 100644 --- a/koregraph/tools/video_builder.py +++ b/koregraph/tools/video_builder.py @@ -76,7 +76,7 @@ def export_choregraphy_keypoints( if export_name is None: export_name = choregraphy.name - export_path = Path(f"temp/{export_name}_soundless.mp4") + export_path = KEYPOINTS_BUILDER_TEMP_DIRECTORY / f"{export_name}_soundless.mp4" with get_writer(export_path, mode="I", fps=60) as video_writer: [ @@ -115,7 +115,7 @@ def keypoints_video_audio_builder_from_choreography(choregraphy: Choregraphy): video = VideoFileClip(str(video_path.absolute())) audio = AudioFileClip(str(aist_file.music.absolute())) - final_filename = Path(f"temp/{choregraphy.name}.mp4") + final_filename = KEYPOINTS_BUILDER_TEMP_DIRECTORY / f"{choregraphy.name}.mp4" final_filename.unlink(missing_ok=True) final_video: VideoClip = video.set_audio(audio) From 1fa4b9d461554deace77fa2838f7b9414fdaf2b9 Mon Sep 17 00:00:00 2001 From: kelly anne Date: Wed, 5 Jun 2024 16:18:35 +0200 Subject: [PATCH 03/14] fix useless import --- koregraph/api/machine_learning/neural_network.py | 11 ++++++----- koregraph/api/posture_proc.py | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/koregraph/api/machine_learning/neural_network.py b/koregraph/api/machine_learning/neural_network.py index 4cfd144..de24318 100644 --- a/koregraph/api/machine_learning/neural_network.py +++ b/koregraph/api/machine_learning/neural_network.py @@ -4,7 +4,7 @@ from typing import List -from keras.layers import Dense, LSTM, Normalization, Dropout +from keras.layers import Dense, LSTM, Normalization, Dropout, Bidirectional from keras.models import Sequential, Model @@ -25,14 +25,15 @@ def prepare_model(X, y) -> Model: return Sequential( [ normalization_layer, - LSTM(256), + Bidirectional(LSTM(256, activation="relu", return_sequences=True)), + Bidirectional(LSTM(128, activation='relu')), Dense(256, activation="relu"), Dense(128, activation="relu"), Dense(64, activation="relu"), - Dropout(rate=0.5), + Dropout(rate=0.2), Dense(64, activation="relu"), - Dropout(rate=0.5), - Dense(y.shape[1], activation="linear"), + Dropout(rate=0.2), + Dense(y.shape[1], activation="sigmoid"), ] ) diff --git a/koregraph/api/posture_proc.py b/koregraph/api/posture_proc.py index 23676ad..f26afe3 100644 --- a/koregraph/api/posture_proc.py +++ b/koregraph/api/posture_proc.py @@ -1,7 +1,6 @@ from numpy import array, ndarray, nan_to_num from koregraph.params import KEYPOINTS_DIRECTORY, FRAME_FORMAT from pickle import load as load_pickle -from numpy import isnan from typing import Tuple From 287f10aaa3f067209ec2660a71d0248d6a989133 Mon Sep 17 00:00:00 2001 From: kelly anne Date: Wed, 5 Jun 2024 20:48:52 +0200 Subject: [PATCH 04/14] Use fill forward to replace missing frames --- .../api/machine_learning/load_dataset.py | 9 ++- koregraph/api/posture_proc.py | 22 ++++++- notebook/chore_per_song.csv | 62 +++++++++++++++++++ 3 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 notebook/chore_per_song.csv diff --git a/koregraph/api/machine_learning/load_dataset.py b/koregraph/api/machine_learning/load_dataset.py index 7966932..1a7aaed 100644 --- a/koregraph/api/machine_learning/load_dataset.py +++ b/koregraph/api/machine_learning/load_dataset.py @@ -36,9 +36,12 @@ def load_preprocess_dataset() -> tuple[ndarray, ndarray]: ] for file in files: - y_tmp, X_tmp = load_pickle_object(GENERATED_PICKLE_DIRECTORY / file) - X = append(X, X_tmp, axis=0) - y = append(y, y_tmp, axis=0) + try: + y_tmp, X_tmp = load_pickle_object(GENERATED_PICKLE_DIRECTORY / file) + X = append(X, X_tmp, axis=0) + y = append(y, y_tmp, axis=0) + except: + print('File does not exist') return X, y diff --git a/koregraph/api/posture_proc.py b/koregraph/api/posture_proc.py index f26afe3..a41f042 100644 --- a/koregraph/api/posture_proc.py +++ b/koregraph/api/posture_proc.py @@ -1,9 +1,27 @@ -from numpy import array, ndarray, nan_to_num +from numpy import array, ndarray, nan_to_num, asarray, isnan from koregraph.params import KEYPOINTS_DIRECTORY, FRAME_FORMAT from pickle import load as load_pickle from typing import Tuple +def fill_forward(arr): + """ + Fill NaN values in the array with the previous row's values for each column. + + Parameters: + arr (numpy.ndarray): Input array with possible NaN values. + + Returns: + numpy.ndarray: Array with NaN values filled forward. + """ + arr = asarray(arr, dtype=float) + + for i in range(1, arr.shape[0]): + mask = isnan(arr[i, :]) + arr[i, mask] = arr[i - 1, mask] + + return arr + def generate_posture_array( choregraphy_name: str, frame_format: tuple = FRAME_FORMAT ) -> array: @@ -18,7 +36,7 @@ def generate_posture_array( with open(KEYPOINTS_DIRECTORY / (choregraphy_name), "rb") as f: data = load_pickle(f) postures = data["keypoints2d"][0, :, :, :2] - postures = nan_to_num(postures, 0) + postures = fill_forward(postures) postures[:, :, 0] = postures[:, :, 0] / frame_format[0] postures[:, :, 1] = postures[:, :, 1] / frame_format[1] diff --git a/notebook/chore_per_song.csv b/notebook/chore_per_song.csv new file mode 100644 index 0000000..f1f0c6a --- /dev/null +++ b/notebook/chore_per_song.csv @@ -0,0 +1,62 @@ +4,name +mBR0,gBR_sFM_cAll_d04_mBR0_ch01.pkl +mBR1,gBR_sFM_cAll_d04_mBR1_ch02.pkl +mBR2,gBR_sFM_cAll_d06_mBR2_ch16.pkl +mBR3,gBR_sFM_cAll_d06_mBR3_ch17.pkl +mBR4,gBR_sFM_cAll_d05_mBR4_ch13.pkl +mBR5,gBR_sFM_cAll_d05_mBR5_ch12.pkl +mHO0,gHO_sFM_cAll_d20_mHO0_ch08.pkl +mHO1,gHO_sFM_cAll_d20_mHO1_ch09.pkl +mHO2,gHO_sFM_cAll_d19_mHO2_ch07.pkl +mHO3,gHO_sFM_cAll_d21_mHO3_ch21.pkl +mHO4,gHO_sFM_cAll_d21_mHO4_ch19.pkl +mHO5,gHO_sFM_cAll_d20_mHO5_ch13.pkl +mJB0,gJB_sFM_cAll_d07_mJB0_ch01.pkl +mJB1,gJB_sFM_cAll_d07_mJB1_ch02.pkl +mJB2,gJB_sFM_cAll_d08_mJB2_ch10.pkl +mJB3,gJB_sFM_cAll_d08_mJB3_ch11.pkl +mJB4,gJB_sFM_cAll_d09_mJB4_ch19.pkl +mJB5,gJB_sFM_cAll_d08_mJB5_ch14.pkl +mJS0,gJS_sFM_cAll_d01_mJS0_ch01.pkl +mJS1,gJS_sFM_cAll_d02_mJS1_ch02.pkl +mJS2,gJS_sFM_cAll_d03_mJS2_ch03.pkl +mJS3,gJS_sFM_cAll_d03_mJS3_ch04.pkl +mJS4,gJS_sFM_cAll_d02_mJS4_ch09.pkl +mJS5,gJS_sFM_cAll_d03_mJS5_ch13.pkl +mKK1,gJB_sFM_cAll_d07_mKK1_ch01.pkl +mKR0,gKR_sFM_cAll_d30_mKR0_ch15.pkl +mKR1,gKR_sFM_cAll_d30_mKR1_ch16.pkl +mKR2,gKR_sFM_cAll_d28_mKR2_ch03.pkl +mKR3,gKR_sFM_cAll_d30_mKR3_ch18.pkl +mKR4,gKR_sFM_cAll_d28_mKR4_ch05.pkl +mKR5,gKR_sFM_cAll_d28_mKR5_ch06.pkl +mLH0,gLH_sFM_cAll_d17_mLH0_ch14.pkl +mLH1,gLH_sFM_cAll_d18_mLH1_ch16.pkl +mLH2,gLH_sFM_cAll_d18_mLH2_ch17.pkl +mLH3,gLH_sFM_cAll_d16_mLH3_ch04.pkl +mLH4,gLH_sFM_cAll_d16_mLH4_ch05.pkl +mLH5,gLH_sFM_cAll_d16_mLH5_ch06.pkl +mLO0,gLO_sFM_cAll_d13_mLO0_ch01.pkl +mLO1,gLO_sFM_cAll_d13_mLO1_ch02.pkl +mLO2,gLO_sFM_cAll_d15_mLO2_ch17.pkl +mLO3,gLO_sFM_cAll_d14_mLO3_ch11.pkl +mLO4,gLO_sFM_cAll_d15_mLO4_ch19.pkl +mLO5,gLO_sFM_cAll_d13_mLO5_ch06.pkl +mMH0,gMH_sFM_cAll_d23_mMH0_ch08.pkl +mMH1,gMH_sFM_cAll_d23_mMH1_ch09.pkl +mMH2,gMH_sFM_cAll_d24_mMH2_ch17.pkl +mMH3,gMH_sFM_cAll_d23_mMH3_ch11.pkl +mMH4,gMH_sFM_cAll_d23_mMH4_ch12.pkl +mMH5,gMH_sFM_cAll_d23_mMH5_ch13.pkl +mPO0,gPO_sFM_cAll_d11_mPO0_ch08.pkl +mPO1,gPO_sFM_cAll_d11_mPO1_ch09.pkl +mPO2,gPO_sFM_cAll_d11_mPO2_ch10.pkl +mPO3,gPO_sFM_cAll_d11_mPO3_ch11.pkl +mPO4,gPO_sFM_cAll_d11_mPO4_ch12.pkl +mPO5,gPO_sFM_cAll_d11_mPO5_ch13.pkl +mWA0,gWA_sFM_cAll_d25_mWA0_ch01.pkl +mWA1,gWA_sFM_cAll_d25_mWA1_ch02.pkl +mWA2,gWA_sFM_cAll_d27_mWA2_ch17.pkl +mWA3,gWA_sFM_cAll_d27_mWA3_ch18.pkl +mWA4,gWA_sFM_cAll_d26_mWA4_ch12.pkl +mWA5,gWA_sFM_cAll_d27_mWA5_ch20.pkl From 89726b426fcb7ef615ffe1ec2a2f0f20d844c569 Mon Sep 17 00:00:00 2001 From: kelly anne Date: Thu, 6 Jun 2024 12:06:24 +0200 Subject: [PATCH 05/14] train using chunks, rename chunk to chunk_api --- koregraph/api/{chunks.py => chunks_api.py} | 4 +- .../api/machine_learning/chunks_workflow.py | 49 +++++++++++++++ .../api/machine_learning/load_dataset.py | 62 +++++++++++++++++-- .../api/machine_learning/neural_network.py | 47 +++++++++++++- .../api/machine_learning/pickle_creation.py | 11 ++++ .../machine_learning/prediction_workflow.py | 15 +++-- koregraph/api/posture_proc.py | 2 +- koregraph/cli/chunk_command.py | 2 +- koregraph/cli/generate_features.py | 20 +++++- koregraph/cli/model.py | 8 ++- koregraph/cli/predict.py | 13 +--- koregraph/params.py | 2 +- 12 files changed, 203 insertions(+), 32 deletions(-) rename koregraph/api/{chunks.py => chunks_api.py} (93%) create mode 100644 koregraph/api/machine_learning/chunks_workflow.py diff --git a/koregraph/api/chunks.py b/koregraph/api/chunks_api.py similarity index 93% rename from koregraph/api/chunks.py rename to koregraph/api/chunks_api.py index 657a7c6..c1276c8 100644 --- a/koregraph/api/chunks.py +++ b/koregraph/api/chunks_api.py @@ -24,10 +24,10 @@ def generate_chunk( choregraphy_name: str, chunk_size: int = CHUNK_SIZE, reload_music: bool = False ): # Clean previous chunks out if needed - chore_path = GENERATED_KEYPOINTS_DIRECTORY / choregraphy_name / chunk_size + chore_path = GENERATED_KEYPOINTS_DIRECTORY / choregraphy_name / str(chunk_size) reset_chunks(chore_path) _, _, _, _, music_name, _ = choregraphy_name.split("_") - music_path = GENERATED_AUDIO_DIRECTORY / music_name / chunk_size + music_path = GENERATED_AUDIO_DIRECTORY / music_name / str(chunk_size) reset_chunks(music_path, reload_music) # Get and save chunks diff --git a/koregraph/api/machine_learning/chunks_workflow.py b/koregraph/api/machine_learning/chunks_workflow.py new file mode 100644 index 0000000..155f9b1 --- /dev/null +++ b/koregraph/api/machine_learning/chunks_workflow.py @@ -0,0 +1,49 @@ +from numpy import expand_dims, float32, ndarray, isnan, any, isinf + +from koregraph.api.machine_learning.neural_network import initialize_model_chunks +from koregraph.api.machine_learning.load_dataset import ( + load_chunk_preprocess_dataset, +) +from koregraph.api.machine_learning.callbacks import BackupCallback +from koregraph.utils.pickle import save_object_pickle +from sklearn.preprocessing import MinMaxScaler + + +def train_chunks_workflow(model_name: str = "model"): + + X, y = load_chunk_preprocess_dataset() + + # scaler = MinMaxScaler() + # X_scaled = scaler.fit_transform(X.reshape(-1, 128)).reshape(-1, 600, 128) + + # X = X.T + # # X = X.reshape((128, 1, -1)) + + y = y.astype(float32) + print("y has nan", isnan(y).any()) + print("X has nan", isnan(X).any()) + print("y has inf", isinf(y).any()) + print("X has inf", isinf(X).any()) + print("Y min", y.min()) + print("Y max", y.max()) + + print("Model X shape:", X.shape) + print("Model y shape:", y.shape) + # return + model = initialize_model_chunks(X, y) + + history = model.fit( + x=X, + y=y, + validation_split=0.2, + batch_size=16, + epochs=50, + # callbacks=[BackupCallback], + ) + + save_object_pickle(model, model_name) + save_object_pickle(history, model_name + "_history") + + +if __name__ == "__main__": + train_chunks_workflow() diff --git a/koregraph/api/machine_learning/load_dataset.py b/koregraph/api/machine_learning/load_dataset.py index 1a7aaed..4186b25 100644 --- a/koregraph/api/machine_learning/load_dataset.py +++ b/koregraph/api/machine_learning/load_dataset.py @@ -2,15 +2,26 @@ All utilities functions to load the train dataset. """ -from typing import Any +from typing import Any, Tuple +from os import listdir -from numpy import ndarray, append +from numpy import ndarray, append, isnan, any, nan_to_num from koregraph.utils.pickle import load_pickle_object -from koregraph.params import GENERATED_PICKLE_DIRECTORY +from koregraph.api.music_to_numpy import music_to_numpy +from koregraph.api.posture_proc import fill_forward +from koregraph.params import ( + GENERATED_KEYPOINTS_DIRECTORY, + GENERATED_AUDIO_DIRECTORY, + GENERATED_PICKLE_DIRECTORY, + ALL_ADVANCED_MOVE_NAMES, + CHUNK_SIZE, + FRAME_FORMAT, +) +from koregraph.models.aist_file import AISTFile -def load_preprocess_dataset() -> tuple[ndarray, ndarray]: +def load_preprocess_dataset() -> Tuple[ndarray, ndarray]: """ Load and preprocess the dataset. @@ -40,10 +51,49 @@ def load_preprocess_dataset() -> tuple[ndarray, ndarray]: y_tmp, X_tmp = load_pickle_object(GENERATED_PICKLE_DIRECTORY / file) X = append(X, X_tmp, axis=0) y = append(y, y_tmp, axis=0) - except: - print('File does not exist') + except Exception as e: + print(f"Error {e}") return X, y +def load_chunk_preprocess_dataset() -> Tuple[ndarray, ndarray]: + chore_names = ALL_ADVANCED_MOVE_NAMES[:100] + X = None + y = None + for chore_name in chore_names: + chore_name = chore_name.replace(".pkl", "") + _, _, _, _, music_name, _ = chore_name.split("_") + + chore_path = GENERATED_KEYPOINTS_DIRECTORY / chore_name / str(CHUNK_SIZE) + music_path = GENERATED_AUDIO_DIRECTORY / music_name / str(CHUNK_SIZE) + print(f"Parsing {chore_name} chunks") + + for file in listdir(chore_path): + chunk_id = file.replace(".pkl", "").split("_")[-1] + chore_filepath = chore_path / file + music_filepath = music_path / f"{music_name}_{chunk_id}.mp3" + + X_tmp = music_to_numpy(music_filepath) + y_tmp = load_pickle_object(chore_filepath)["keypoints2d"] + + y_tmp = fill_forward(y_tmp) + if isnan(y_tmp).any(): + print(f"Fill forward failed for chunk {chunk_id}. Filling with 0") + y_tmp = nan_to_num(y_tmp, 0) + y_tmp[:, :, 0] = y_tmp[:, :, 0] / FRAME_FORMAT[0] + y_tmp[:, :, 1] = y_tmp[:, :, 1] / FRAME_FORMAT[1] + + if y is None: + y = y_tmp + else: + y = append(y, y_tmp, axis=0) + if X is None: + X = X_tmp + else: + X = append(X, X_tmp, axis=0) + + return X.reshape(-1, CHUNK_SIZE * 60, 128), y.reshape(-1, CHUNK_SIZE * 60 * 34) + + def check_dataset_format(): ... diff --git a/koregraph/api/machine_learning/neural_network.py b/koregraph/api/machine_learning/neural_network.py index de24318..725da35 100644 --- a/koregraph/api/machine_learning/neural_network.py +++ b/koregraph/api/machine_learning/neural_network.py @@ -4,8 +4,16 @@ from typing import List -from keras.layers import Dense, LSTM, Normalization, Dropout, Bidirectional +from keras.layers import ( + Dense, + LSTM, + Normalization, + Dropout, + Bidirectional, + BatchNormalization, +) from keras.models import Sequential, Model +from keras.optimizers import Adam def prepare_model(X, y) -> Model: @@ -26,7 +34,7 @@ def prepare_model(X, y) -> Model: [ normalization_layer, Bidirectional(LSTM(256, activation="relu", return_sequences=True)), - Bidirectional(LSTM(128, activation='relu')), + Bidirectional(LSTM(128, activation="relu")), Dense(256, activation="relu"), Dense(128, activation="relu"), Dense(64, activation="relu"), @@ -63,5 +71,40 @@ def initialize_model(X, y) -> Model: return compiled_model +def initialize_model_chunks(X, y) -> Model: + """Initialize a compiled model. + + Returns: + Model: The compiled model. + """ + + normalization_layer = Normalization() + normalization_layer.adapt(X) + + new_model = Sequential( + [ + normalization_layer, + Bidirectional( + LSTM( + 128, activation="tanh", return_sequences=True, recurrent_dropout=0.2 + ) + ), + BatchNormalization(), + Bidirectional(LSTM(64, activation="tanh", recurrent_dropout=0.2)), + Dense(256, activation="relu", activity_regularizer="l2"), + Dense(128, activation="relu", activity_regularizer="l2"), + Dense(64, activation="relu", activity_regularizer="l2"), + Dropout(rate=0.2), + Dense(64, activation="relu", activity_regularizer="l2"), + Dropout(rate=0.2), + Dense(y.shape[1], activation="sigmoid"), + ] + ) + + adam = Adam(learning_rate=0.00001, clipvalue=0.01) + new_model.compile(loss="mse", optimizer=adam, metrics=["mae"]) + return new_model + + if __name__ == "__main__": initialize_model() diff --git a/koregraph/api/machine_learning/pickle_creation.py b/koregraph/api/machine_learning/pickle_creation.py index 8afdef8..68b5269 100644 --- a/koregraph/api/machine_learning/pickle_creation.py +++ b/koregraph/api/machine_learning/pickle_creation.py @@ -1,12 +1,14 @@ from pickle import dump as dump_pickle from os import makedirs +from koregraph.api.chunks_api import generate_chunk from koregraph.api.music_to_numpy import music_to_numpy from koregraph.api.posture_proc import generate_posture_array from koregraph.params import ( AUDIO_DIRECTORY, GENERATED_PICKLE_DIRECTORY, ALL_ADVANCED_MOVE_NAMES, + CHUNK_SIZE, ) @@ -44,5 +46,14 @@ def generate_pickle_files(): continue +def generate_all_chunks(): + for chore_name in ALL_ADVANCED_MOVE_NAMES: + try: + generate_chunk(chore_name.replace(".pkl", ""), CHUNK_SIZE, False) + except Exception as e: + print(f"Error with {chore_name}: {e}") + continue + + if __name__ == "__main__": generate_pickle_files() diff --git a/koregraph/api/machine_learning/prediction_workflow.py b/koregraph/api/machine_learning/prediction_workflow.py index bf0caa3..13b7145 100644 --- a/koregraph/api/machine_learning/prediction_workflow.py +++ b/koregraph/api/machine_learning/prediction_workflow.py @@ -14,22 +14,25 @@ ) -def predict(audio_name: str = "mBR0", model_name: str = "model", chore_id: str = "01"): +def predict(audio_name: str = "mBR0", model_name: str = "model", chunk: bool = False): model_path = MODEL_OUTPUT_DIRECTORY / (model_name + ".pkl") model = load_pickle_object(model_path) audio_filepath = AUDIO_DIRECTORY / (audio_name + ".mp3") input = music_to_numpy(audio_filepath) - # TODO remove this step when reshape is done in preprocessing workflow - input = input.reshape(-1, 1, input.shape[1]) + if chunk: + print("input shape", input.shape) + input = input.reshape(1, input.shape[0], input.shape[1]) + else: + input = input.reshape(-1, 1, input.shape[1]) + print(input.shape) prediction = model.predict(input) prediction = upscale_posture_pred(prediction) - print(prediction.shape) - + print("Prediction shape:", prediction.shape) prediction_name = ( - model_name.replace("_", "") + "_sBM_cAll_d00_" + audio_name + f"_ch{chore_id}" + model_name.replace("_", "-") + "_sBM_cAll_d00_" + audio_name + "_ch01" ) chore = Choregraphy( diff --git a/koregraph/api/posture_proc.py b/koregraph/api/posture_proc.py index a41f042..2ea2bf8 100644 --- a/koregraph/api/posture_proc.py +++ b/koregraph/api/posture_proc.py @@ -22,6 +22,7 @@ def fill_forward(arr): return arr + def generate_posture_array( choregraphy_name: str, frame_format: tuple = FRAME_FORMAT ) -> array: @@ -43,7 +44,6 @@ def generate_posture_array( return postures.reshape(-1, 34) -# @TODO: verifier le format d'entree de prediction def upscale_posture_pred( prediction: ndarray, frame_format: tuple = FRAME_FORMAT ) -> array: diff --git a/koregraph/cli/chunk_command.py b/koregraph/cli/chunk_command.py index f224aa7..71601a4 100644 --- a/koregraph/cli/chunk_command.py +++ b/koregraph/cli/chunk_command.py @@ -2,7 +2,7 @@ from pathlib import Path from koregraph.params import CHUNK_SIZE -from koregraph.api.chunks import generate_chunk +from koregraph.api.chunks_api import generate_chunk parser = ArgumentParser( "Koregraph chunk", diff --git a/koregraph/cli/generate_features.py b/koregraph/cli/generate_features.py index 5b22306..049bde0 100644 --- a/koregraph/cli/generate_features.py +++ b/koregraph/cli/generate_features.py @@ -1,8 +1,24 @@ -from koregraph.api.machine_learning.pickle_creation import generate_pickle_files +from argparse import ArgumentParser + +from koregraph.api.machine_learning.pickle_creation import ( + generate_pickle_files, + generate_all_chunks, +) + +parser = ArgumentParser( + "Generate", + description="Use this generate features pickle files", +) + +parser.add_argument("--chunks", dest="chunks", action="store_true") def main(): - generate_pickle_files() + args = parser.parse_args() + if args.chunks: + generate_all_chunks() + else: + generate_pickle_files() if __name__ == "__main__": diff --git a/koregraph/cli/model.py b/koregraph/cli/model.py index 1fc7c19..7dff665 100644 --- a/koregraph/cli/model.py +++ b/koregraph/cli/model.py @@ -6,6 +6,7 @@ from tensorflow.python.keras.models import Model from koregraph.api.machine_learning.workflow import train_workflow +from koregraph.api.machine_learning.chunks_workflow import train_chunks_workflow from koregraph.params import MODEL_OUTPUT_DIRECTORY, AUDIO_DIRECTORY @@ -23,12 +24,17 @@ default="model", ) +parser.add_argument("--chunks", dest="chunks", action="store_true") + def main(): arguments = parser.parse_args() model_name = arguments.model_name - train_workflow(model_name=model_name) + if arguments.chunks: + train_chunks_workflow(model_name=model_name) + else: + train_workflow(model_name=model_name) if __name__ == "__main__": diff --git a/koregraph/cli/predict.py b/koregraph/cli/predict.py index 75df81c..170da49 100644 --- a/koregraph/cli/predict.py +++ b/koregraph/cli/predict.py @@ -20,14 +20,7 @@ default="model", ) -parser.add_argument( - "-i", - "--chore-id", - dest="chore_id", - required=False, - default="01", - help="Id of the chore to be created. Ex: '01'", -) +parser.add_argument("--chunks", dest="is_chunks", action="store_true") def main(): @@ -35,9 +28,9 @@ def main(): audio_name = arguments.audio_name model_name = arguments.model_name - chore_id = arguments.chore_id + is_chunks = arguments.is_chunks - predict_api(audio_name=audio_name, model_name=model_name, chore_id=chore_id) + predict_api(audio_name=audio_name, model_name=model_name, chunk=is_chunks) if __name__ == "__main__": diff --git a/koregraph/params.py b/koregraph/params.py index 201fb5b..73cdbf0 100644 --- a/koregraph/params.py +++ b/koregraph/params.py @@ -30,7 +30,7 @@ environ.get("GENERATED_AUDIO_DIRECTORY", "generated/chunks/music") ) -CHUNK_SIZE: int = int(environ.get("CHUNK_SIZE", "10")) +CHUNK_SIZE: int = int(environ.get("CHUNK_SIZE", "5")) if not AUDIO_DIRECTORY.exists(): raise FileNotFoundError( From 5add198c050bd7310510ee32054048390929078a Mon Sep 17 00:00:00 2001 From: kelly anne Date: Thu, 6 Jun 2024 12:18:19 +0200 Subject: [PATCH 06/14] merge master generate cli --- koregraph/cli/generate_features.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/koregraph/cli/generate_features.py b/koregraph/cli/generate_features.py index f974d25..049bde0 100644 --- a/koregraph/cli/generate_features.py +++ b/koregraph/cli/generate_features.py @@ -1,4 +1,3 @@ -<<<<<<< HEAD from argparse import ArgumentParser from koregraph.api.machine_learning.pickle_creation import ( @@ -20,13 +19,6 @@ def main(): generate_all_chunks() else: generate_pickle_files() -======= -from koregraph.api.machine_learning.pickle_creation import generate_pickle_files - - -def main(): - generate_pickle_files() ->>>>>>> 2b3865a67773b463ee9dbded945f4c623e35febe if __name__ == "__main__": From c923ef06b136948f41b0bcc3eb9c1d3504960afb Mon Sep 17 00:00:00 2001 From: Kelly-an <166104677+Kelly-an@users.noreply.github.com> Date: Mon, 10 Jun 2024 09:47:37 +0200 Subject: [PATCH 07/14] Feature/predict next moves (#61) * working chunk model, takes 4s chore, predicts 1s * updated all advance move name var for vm * backup * add backup * ci cd --- .../api/machine_learning/load_dataset.py | 70 +++++- .../api/machine_learning/neural_network.py | 70 ++++-- .../machine_learning/prediction_workflow.py | 90 +++++++- .../{ => train_workflow}/chunks_workflow.py | 0 .../train_workflow/next_chunks_workflow.py | 65 ++++++ .../{ => train_workflow}/workflow.py | 19 +- koregraph/api/posture_proc.py | 21 +- koregraph/cli/model.py | 15 +- koregraph/cli/predict.py | 31 ++- koregraph/params.py | 18 ++ koregraph/utils/preproc.py | 7 + notebook/all_advanced_move_names.csv | 212 ++++++++++++++++++ 12 files changed, 590 insertions(+), 28 deletions(-) rename koregraph/api/machine_learning/{ => train_workflow}/chunks_workflow.py (100%) create mode 100644 koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py rename koregraph/api/machine_learning/{ => train_workflow}/workflow.py (60%) create mode 100644 koregraph/utils/preproc.py create mode 100644 notebook/all_advanced_move_names.csv diff --git a/koregraph/api/machine_learning/load_dataset.py b/koregraph/api/machine_learning/load_dataset.py index 4186b25..2ed43c8 100644 --- a/koregraph/api/machine_learning/load_dataset.py +++ b/koregraph/api/machine_learning/load_dataset.py @@ -5,9 +5,9 @@ from typing import Any, Tuple from os import listdir -from numpy import ndarray, append, isnan, any, nan_to_num +from numpy import ndarray, append, isnan, any, nan_to_num, concatenate, split, delete -from koregraph.utils.pickle import load_pickle_object +from koregraph.utils.pickle import load_pickle_object, save_object_pickle from koregraph.api.music_to_numpy import music_to_numpy from koregraph.api.posture_proc import fill_forward from koregraph.params import ( @@ -17,8 +17,10 @@ ALL_ADVANCED_MOVE_NAMES, CHUNK_SIZE, FRAME_FORMAT, + GENERATED_FEATURES_DIRECTORY, + PERCENTAGE_CUT, ) -from koregraph.models.aist_file import AISTFile +from koregraph.utils.preproc import cut_percentage def load_preprocess_dataset() -> Tuple[ndarray, ndarray]: @@ -81,8 +83,8 @@ def load_chunk_preprocess_dataset() -> Tuple[ndarray, ndarray]: if isnan(y_tmp).any(): print(f"Fill forward failed for chunk {chunk_id}. Filling with 0") y_tmp = nan_to_num(y_tmp, 0) - y_tmp[:, :, 0] = y_tmp[:, :, 0] / FRAME_FORMAT[0] - y_tmp[:, :, 1] = y_tmp[:, :, 1] / FRAME_FORMAT[1] + # y_tmp[:, :, 0] = y_tmp[:, :, 0] / FRAME_FORMAT[0] + # y_tmp[:, :, 1] = y_tmp[:, :, 1] / FRAME_FORMAT[1] if y is None: y = y_tmp @@ -96,4 +98,62 @@ def load_chunk_preprocess_dataset() -> Tuple[ndarray, ndarray]: return X.reshape(-1, CHUNK_SIZE * 60, 128), y.reshape(-1, CHUNK_SIZE * 60 * 34) +def load_next_chunks_preprocess_dataset(perc_cut: float = PERCENTAGE_CUT): + chore_names = ALL_ADVANCED_MOVE_NAMES[:100] + X = None + y = None + for chore_name in chore_names: + chore_name = chore_name.replace(".pkl", "") + _, _, _, _, music_name, _ = chore_name.split("_") + + chore_path = GENERATED_KEYPOINTS_DIRECTORY / chore_name / str(CHUNK_SIZE) + music_path = GENERATED_AUDIO_DIRECTORY / music_name / str(CHUNK_SIZE) + print(f"Parsing {chore_name} chunks") + + for file in listdir(chore_path): + chunk_id = file.replace(".pkl", "").split("_")[-1] + chore_filepath = chore_path / file + music_filepath = music_path / f"{music_name}_{chunk_id}.mp3" + + audio_tmp = music_to_numpy(music_filepath) + chore_tmp = load_pickle_object(chore_filepath)["keypoints2d"] + + chore_tmp = fill_forward(chore_tmp) + if isnan(chore_tmp).any(): + print(f"Fill forward failed for chunk {chunk_id}. Filling with 0") + chore_tmp = nan_to_num(chore_tmp, 0) + chore_tmp[:, :, 0] = chore_tmp[:, :, 0] / FRAME_FORMAT[0] + chore_tmp[:, :, 1] = chore_tmp[:, :, 1] / FRAME_FORMAT[1] + + chore_X, y_tmp = cut_percentage(chore_tmp.reshape(-1, 34), perc_cut) + audio_X, _ = cut_percentage(audio_tmp, perc_cut) + # X_tmp = concatenate((chore_X, audio_X), axis=1) + X_tmp = chore_X + + # print('X_tmp shape', X_tmp.shape) + # print('y_tmp shape', y_tmp.shape) + + if y is None: + y = y_tmp + else: + y = append(y, y_tmp, axis=0) + + if X is None: + X = X_tmp + else: + X = append(X, X_tmp, axis=0) + + X = X.reshape(-1, int(CHUNK_SIZE * (1 - perc_cut)) * 60, X.shape[-1]) + y = y.reshape(-1, int(CHUNK_SIZE * perc_cut) * 60 * 34) + + print("X final shape", X.shape) + print("y final shape", y.shape) + save_object_pickle(X, GENERATED_FEATURES_DIRECTORY / "x") + save_object_pickle(y, GENERATED_FEATURES_DIRECTORY / "y") + + y = delete(y, [60, 63], axis=0) + X = delete(X, [60, 63], axis=0) + return X, y + + def check_dataset_format(): ... diff --git a/koregraph/api/machine_learning/neural_network.py b/koregraph/api/machine_learning/neural_network.py index 2388da6..6ddddcd 100644 --- a/koregraph/api/machine_learning/neural_network.py +++ b/koregraph/api/machine_learning/neural_network.py @@ -10,8 +10,12 @@ Normalization, Dropout, Bidirectional, + Conv2D, BatchNormalization, + Flatten, + Input, ) +from keras.initializers import glorot_uniform from keras.models import Sequential, Model from keras.optimizers import Adam @@ -32,7 +36,7 @@ def prepare_model(X, y) -> Model: return Sequential( [ - normalization_layer, + # normalization_layer, Bidirectional(LSTM(256, activation="relu", return_sequences=True)), Bidirectional(LSTM(128, activation="relu", return_sequences=True)), Bidirectional(LSTM(64, activation="relu")), @@ -44,7 +48,7 @@ def prepare_model(X, y) -> Model: Dropout(rate=0.2), Dense(32, activation="relu"), Dropout(rate=0.2), - Dense(y.shape[1], activation="sigmoid"), + Dense(y.shape[1], activation="relu"), ] ) @@ -81,33 +85,73 @@ def initialize_model_chunks(X, y) -> Model: Model: The compiled model. """ - normalization_layer = Normalization() - normalization_layer.adapt(X) - + # normalization_layer = Normalization() + # normalization_layer.adapt(X) + print("x 0 shape", X[0].shape) new_model = Sequential( [ - normalization_layer, + # normalization_layer, Bidirectional( LSTM( - 128, activation="tanh", return_sequences=True, recurrent_dropout=0.2 - ) + 256, + activation="tanh", + kernel_initializer=glorot_uniform(), + return_sequences=True, + ), ), - BatchNormalization(), - Bidirectional(LSTM(64, activation="tanh", recurrent_dropout=0.2)), + # # BatchNormalization(), + LSTM(128, activation="tanh", recurrent_dropout=0.2), + Dense(256, activation="relu"), Dense(256, activation="relu", activity_regularizer="l2"), Dense(128, activation="relu", activity_regularizer="l2"), - Dense(64, activation="relu", activity_regularizer="l2"), Dropout(rate=0.2), Dense(64, activation="relu", activity_regularizer="l2"), Dropout(rate=0.2), - Dense(y.shape[1], activation="sigmoid"), + Dense(y.shape[1], activation="linear"), ] ) - adam = Adam(learning_rate=0.00001, clipvalue=0.01) + adam = Adam(learning_rate=0.00001) # , clipvalue=0.01) new_model.compile(loss="mse", optimizer=adam, metrics=["mae"]) return new_model +def initialize_model_next_chunks(X, y) -> Model: + """Initialize a compiled model. + + Returns: + Model: The compiled model. + """ + + # normalization_layer = Normalization() + # normalization_layer.adapt(X) + print("x 0 shape", X[0].shape) + new_model = Sequential( + [ + # Input(), + Conv2D( + 258, + kernel_size=(3, 3), + activation="relu", + input_shape=X[0].shape, + padding="same", + ), + # Dropout(rate=0.2), + # Conv2D(128, kernel_size=(3, 3), activation="relu", padding="same"), + BatchNormalization(), + Flatten(), + Dense(256, activation="relu"), + Dense(128, activation="relu"), + Dropout(rate=0.2), + Dense(64, activation="relu"), # , activity_regularizer="l2"), + Dropout(rate=0.2), + Dense(y.shape[1], activation="relu"), + ] + ) + + new_model.compile(loss="mse", optimizer="adam", metrics=["mae"]) + return new_model + + if __name__ == "__main__": initialize_model() diff --git a/koregraph/api/machine_learning/prediction_workflow.py b/koregraph/api/machine_learning/prediction_workflow.py index a616d1f..7c6ab0c 100644 --- a/koregraph/api/machine_learning/prediction_workflow.py +++ b/koregraph/api/machine_learning/prediction_workflow.py @@ -1,4 +1,4 @@ -from numpy import ones as np_ones +from numpy import ones as np_ones, concatenate, append from koregraph.models.choregraphy import Choregraphy from koregraph.managers.choregraphy import save_choregaphy_chunk from koregraph.utils.pickle import load_pickle_object @@ -11,7 +11,13 @@ AUDIO_DIRECTORY, MODEL_OUTPUT_DIRECTORY, PREDICTION_OUTPUT_DIRECTORY, + PERCENTAGE_CUT, + GENERATED_KEYPOINTS_DIRECTORY, + GENERATED_AUDIO_DIRECTORY, + CHUNK_SIZE, + FRAME_FORMAT, ) +from koregraph.utils.preproc import cut_percentage from koregraph.api.audio_proc import scale_audio @@ -27,14 +33,18 @@ def predict(audio_name: str = "mBR0", model_name: str = "model", chunk: bool = F input = input.reshape(1, input.shape[0], input.shape[1]) else: # TODO remove this step when reshape is done in preprocessing workflow - input = scale_audio(input) + # input = scale_audio(input) input = input.reshape(-1, 1, input.shape[1]) + print("input min:", input.min()) + print("input max:", input.max()) print(input.shape) prediction = model.predict(input) - prediction = upscale_posture_pred(prediction) + # prediction = upscale_posture_pred(prediction) print("Prediction shape:", prediction.shape) + print("predciton min", prediction.min()) + print("predciton max", prediction.max()) prediction_name = ( model_name.replace("_", "-") + "_sBM_cAll_d00_" + audio_name + "_ch01" ) @@ -51,5 +61,79 @@ def predict(audio_name: str = "mBR0", model_name: str = "model", chunk: bool = F print("Happy viewing!") +def predict_next_move( + audio_name: str = "mBR0", + model_name: str = "model", + chore_chunk_name: str = "gBR_sFM_cAll_d04_mBR0_ch01", + chunk_id: int = 0, + perc_cut: float = PERCENTAGE_CUT, +): + model_path = MODEL_OUTPUT_DIRECTORY / (model_name + ".pkl") + model = load_pickle_object(model_path) + + audio_filepath = ( + GENERATED_AUDIO_DIRECTORY + / audio_name + / str(CHUNK_SIZE) + / (f"{audio_name}_{chunk_id}.mp3") + ) + audio = music_to_numpy(audio_filepath) + + # cut input, take 8 sec + print("Before cut", audio.shape) + audio, _ = cut_percentage(audio, perc_cut) + print("After cut", audio.shape) + print("min audio: ", audio.min()) + print("max audio: ", audio.max()) + # add beginning of chore + chore = load_pickle_object( + GENERATED_KEYPOINTS_DIRECTORY + / chore_chunk_name + / str(CHUNK_SIZE) + / (f"{chore_chunk_name}_{chunk_id}.pkl") + ) + input = chore["keypoints2d"] + input[:, :, 0] = input[:, :, 0] / FRAME_FORMAT[0] + input[:, :, 1] = input[:, :, 1] / FRAME_FORMAT[1] + input = input.reshape(-1, 34) + input, _ = cut_percentage(input.reshape(-1, 34), perc_cut) + # chore, _ = cut_percentage(chore.reshape(-1, 34), perc_cut) + + print("min input: ", input.min()) + print("max input: ", input.max()) + # input = concatenate((chore, audio), axis=1) + + input = input.reshape(1, 240, 17, 2) + # input = input.reshape(-1, 8160) + print(input.shape) + # print(input) + prediction = model.predict(input) + print("prediction shape", prediction.shape) + print("prediction min", prediction.min()) + print("prediction max", prediction.max()) + prediction = upscale_posture_pred(prediction) + + print("Prediction shape:", prediction.shape) + prediction_name = ( + model_name.replace("_", "-") + "_sBM_cAll_d00_" + audio_name + "_ch01" + ) + + # print('chore') + # print(chore.shape) + # print(chore) + output = append(upscale_posture_pred(input), prediction) + + chore = Choregraphy( + prediction_name, output.reshape(-1, 17, 2), np_ones(output.shape[0]) + ) + # Save prediction to pkl + save_choregaphy_chunk(chore, PREDICTION_OUTPUT_DIRECTORY) + + # Create video + keypoints_video_audio_builder_from_choreography(chore) + + print("Happy viewing!") + + if __name__ == "__main__": predict(audio_name="mBR2", chore_id="02") diff --git a/koregraph/api/machine_learning/chunks_workflow.py b/koregraph/api/machine_learning/train_workflow/chunks_workflow.py similarity index 100% rename from koregraph/api/machine_learning/chunks_workflow.py rename to koregraph/api/machine_learning/train_workflow/chunks_workflow.py diff --git a/koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py b/koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py new file mode 100644 index 0000000..d24eae3 --- /dev/null +++ b/koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py @@ -0,0 +1,65 @@ +from numpy import expand_dims, float32, ndarray, isnan, any, isinf + +from koregraph.api.machine_learning.neural_network import initialize_model_next_chunks +from koregraph.api.machine_learning.load_dataset import ( + load_next_chunks_preprocess_dataset as load_preprocess_dataset, +) +from koregraph.api.machine_learning.callbacks import BackupCallback +from koregraph.utils.pickle import save_object_pickle, load_pickle_object +from sklearn.preprocessing import MinMaxScaler +from koregraph.params import GENERATED_FEATURES_DIRECTORY, CHUNK_SIZE, PERCENTAGE_CUT +from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping +from koregraph.params import WEIGHTS_BACKUP_DIRECTORY + + +def train_workflow(model_name: str = "model"): + + # X, y = load_preprocess_dataset() + X = load_pickle_object(GENERATED_FEATURES_DIRECTORY / "x.pkl") + y = load_pickle_object(GENERATED_FEATURES_DIRECTORY / "y.pkl") + y = y.astype(float32) + + print("y has nan", isnan(y).any()) + print("X has nan", isnan(X).any()) + print("y has inf", isinf(y).any()) + print("X has inf", isinf(X).any()) + print("Y min", y.min()) + print("Y max", y.max()) + + X = X.reshape(-1, int((CHUNK_SIZE * (1 - PERCENTAGE_CUT)) * 60), 17, 2) + y = y.reshape(-1, int(CHUNK_SIZE * PERCENTAGE_CUT * 60 * 17 * 2)) + + print("Model X shape:", X.shape) + print("Model y shape:", y.shape) + + model = initialize_model_next_chunks(X, y) + + history = model.fit( + x=X, + y=y, + validation_split=0.2, + batch_size=16, + epochs=1000, + callbacks=[ + ModelCheckpoint( + WEIGHTS_BACKUP_DIRECTORY / f"{model_name}_backup.keras", + monitor="val_loss", + verbose=0, + save_best_only=False, + save_weights_only=False, + mode="auto", + save_freq="epoch", + initial_value_threshold=None, + ), + EarlyStopping( + monitor="val_loss", patience=7, verbose=0, restore_best_weights=True + ), + ], + ) + + save_object_pickle(model, model_name) + save_object_pickle(history, model_name + "_history") + + +if __name__ == "__main__": + train_workflow() diff --git a/koregraph/api/machine_learning/workflow.py b/koregraph/api/machine_learning/train_workflow/workflow.py similarity index 60% rename from koregraph/api/machine_learning/workflow.py rename to koregraph/api/machine_learning/train_workflow/workflow.py index 9765394..c676866 100644 --- a/koregraph/api/machine_learning/workflow.py +++ b/koregraph/api/machine_learning/train_workflow/workflow.py @@ -9,6 +9,9 @@ from koregraph.utils.pickle import save_object_pickle from sklearn.preprocessing import MinMaxScaler from koregraph.api.audio_proc import scale_audio +from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping + +from koregraph.params import WEIGHTS_BACKUP_DIRECTORY def train_workflow(model_name: str = "model"): @@ -29,7 +32,21 @@ def train_workflow(model_name: str = "model"): validation_split=0.2, epochs=20, batch_size=16, - callbacks=[BackupCallback, StoppingCallback], + callbacks=[ + ModelCheckpoint( + WEIGHTS_BACKUP_DIRECTORY / f"{model_name}_backup.keras", + monitor="val_loss", + verbose=0, + save_best_only=False, + save_weights_only=False, + mode="auto", + save_freq="epoch", + initial_value_threshold=None, + ), + EarlyStopping( + monitor="val_loss", patience=7, verbose=0, restore_best_weights=True + ), + ], ) save_object_pickle(model, model_name) diff --git a/koregraph/api/posture_proc.py b/koregraph/api/posture_proc.py index 397d255..5e4cd21 100644 --- a/koregraph/api/posture_proc.py +++ b/koregraph/api/posture_proc.py @@ -39,12 +39,29 @@ def generate_posture_array( postures = data["keypoints2d"][0, :, :, :2] postures = fill_forward(postures) postures = nan_to_num(postures, 0) - postures[:, :, 0] = postures[:, :, 0] / frame_format[0] - postures[:, :, 1] = postures[:, :, 1] / frame_format[1] return postures.reshape(-1, 34) +def scale_posture_pred( + prediction: ndarray, frame_format: tuple = FRAME_FORMAT +) -> array: + """Create a numpy array with 34 columns + + Args: + name (str): The choregraphy file's name. + + Returns: + Array of positions: The postures 34 columns N rows. + """ + + prediction = prediction.reshape(-1, 17, 2) + prediction[:, :, 0] = prediction[:, :, 0] / frame_format[0] + prediction[:, :, 1] = prediction[:, :, 1] / frame_format[1] + + return prediction + + def upscale_posture_pred( prediction: ndarray, frame_format: tuple = FRAME_FORMAT ) -> array: diff --git a/koregraph/cli/model.py b/koregraph/cli/model.py index 7dff665..97f1729 100644 --- a/koregraph/cli/model.py +++ b/koregraph/cli/model.py @@ -5,8 +5,13 @@ from tensorflow.python.keras.models import Model -from koregraph.api.machine_learning.workflow import train_workflow -from koregraph.api.machine_learning.chunks_workflow import train_chunks_workflow +from koregraph.api.machine_learning.train_workflow.workflow import train_workflow +from koregraph.api.machine_learning.train_workflow.chunks_workflow import ( + train_chunks_workflow, +) +from koregraph.api.machine_learning.train_workflow.next_chunks_workflow import ( + train_workflow as train_pred_next_workflow, +) from koregraph.params import MODEL_OUTPUT_DIRECTORY, AUDIO_DIRECTORY @@ -26,13 +31,19 @@ parser.add_argument("--chunks", dest="chunks", action="store_true") +parser.add_argument("--next-chunks", dest="predict_next", action="store_true") + def main(): arguments = parser.parse_args() model_name = arguments.model_name if arguments.chunks: + print("Training with chunks") train_chunks_workflow(model_name=model_name) + elif arguments.predict_next: + print("Training with chunks: predicting next X seconds") + train_pred_next_workflow(model_name=model_name) else: train_workflow(model_name=model_name) diff --git a/koregraph/cli/predict.py b/koregraph/cli/predict.py index 170da49..29eb686 100644 --- a/koregraph/cli/predict.py +++ b/koregraph/cli/predict.py @@ -1,6 +1,9 @@ from argparse import ArgumentParser -from koregraph.api.machine_learning.prediction_workflow import predict as predict_api +from koregraph.api.machine_learning.prediction_workflow import ( + predict as predict_api, + predict_next_move, +) parser = ArgumentParser( "Koregraph prediction", @@ -20,8 +23,18 @@ default="model", ) +parser.add_argument( + "-c", "--choregraphy", dest="choregraphy", required=False, help="Choregraphy name" +) + +parser.add_argument( + "-i", "--chunk-id", dest="chunk_id", required=False, help="Choregraphy name" +) + parser.add_argument("--chunks", dest="is_chunks", action="store_true") +parser.add_argument("--predict-next", dest="predict_next", action="store_true") + def main(): arguments = parser.parse_args() @@ -29,8 +42,22 @@ def main(): audio_name = arguments.audio_name model_name = arguments.model_name is_chunks = arguments.is_chunks + choregraphy = arguments.choregraphy + chunk_id = arguments.chunk_id - predict_api(audio_name=audio_name, model_name=model_name, chunk=is_chunks) + if is_chunks: + predict_api(audio_name=audio_name, model_name=model_name, chunk=is_chunks) + elif arguments.predict_next: + assert choregraphy is not None + assert chunk_id is not None + predict_next_move( + audio_name=audio_name, + model_name=model_name, + chore_chunk_name=choregraphy, + chunk_id=chunk_id, + ) + else: + predict_api(audio_name=audio_name, model_name=model_name, chunk=is_chunks) if __name__ == "__main__": diff --git a/koregraph/params.py b/koregraph/params.py index b7b5901..c0a2789 100644 --- a/koregraph/params.py +++ b/koregraph/params.py @@ -1,6 +1,7 @@ from os import environ, path, listdir from pathlib import Path from dotenv import load_dotenv +from pickle import load as load_pickle from koregraph.models.constants import LAST_CHUNK_TYPE @@ -97,3 +98,20 @@ X_MIN = -80 X_MAX = 0 + +PERCENTAGE_CUT = 0.2 + +GENERATED_FEATURES_DIRECTORY: Path = PROJECT_ROOT.joinpath( + environ.get( + "GENERATED_FEATURES_DIRECTORY", + "generated/features", + ) +) +GENERATED_FEATURES_DIRECTORY.mkdir(parents=True, exist_ok=True) + +if len(ALL_ADVANCED_MOVE_NAMES) == 0: + try: + with open("data/all_advanced_move_names.pkl", "rb") as f: + ALL_ADVANCED_MOVE_NAMES = load_pickle(f) + except Exception as e: + print(f"Could not load all_advanced_move_names variable: {e}") diff --git a/koregraph/utils/preproc.py b/koregraph/utils/preproc.py new file mode 100644 index 0000000..87e1942 --- /dev/null +++ b/koregraph/utils/preproc.py @@ -0,0 +1,7 @@ +from typing import Tuple +from numpy import ndarray + + +def cut_percentage(x: ndarray, perc: float) -> Tuple[ndarray, ndarray]: + idx = len(x) - int(len(x) * perc) + return x[:idx], x[idx:] diff --git a/notebook/all_advanced_move_names.csv b/notebook/all_advanced_move_names.csv new file mode 100644 index 0000000..1e1aba9 --- /dev/null +++ b/notebook/all_advanced_move_names.csv @@ -0,0 +1,212 @@ +gKR_sFM_cAll_d30_mKR3_ch18.pkl +gMH_sFM_cAll_d24_mMH2_ch17.pkl +gPO_sFM_cAll_d11_mPO2_ch10.pkl +gJB_sFM_cAll_d07_mJB0_ch01.pkl +gLO_sFM_cAll_d15_mLO2_ch17.pkl +gWA_sFM_cAll_d25_mWA1_ch02.pkl +gPO_sFM_cAll_d11_mPO3_ch11.pkl +gJB_sFM_cAll_d07_mKK1_ch01.pkl +gJB_sFM_cAll_d07_mJB1_ch02.pkl +gMH_sFM_cAll_d23_mMH2_ch10.pkl +gJB_sFM_cAll_d08_mJB3_ch11.pkl +gJS_sFM_cAll_d02_mJS4_ch09.pkl +gJS_sFM_cAll_d03_mJS5_ch13.pkl +gHO_sFM_cAll_d20_mHO0_ch08.pkl +gLH_sFM_cAll_d18_mLH2_ch17.pkl +gJS_sFM_cAll_d02_mJS1_ch02.pkl +gHO_sFM_cAll_d20_mHO1_ch09.pkl +gJS_sFM_cAll_d03_mJS4_ch12.pkl +gWA_sFM_cAll_d25_mWA0_ch01.pkl +gJB_sFM_cAll_d08_mJB2_ch10.pkl +gHO_sFM_cAll_d21_mHO4_ch19.pkl +gJS_sFM_cAll_d01_mJS1_ch07.pkl +gMH_sFM_cAll_d23_mMH3_ch11.pkl +gPO_sFM_cAll_d12_mPO2_ch17.pkl +gBR_sFM_cAll_d06_mBR3_ch17.pkl +gHO_sFM_cAll_d20_mHO5_ch13.pkl +gLH_sFM_cAll_d16_mLH4_ch05.pkl +gLH_sFM_cAll_d17_mLH0_ch14.pkl +gHO_sFM_cAll_d21_mHO1_ch16.pkl +gHO_sFM_cAll_d20_mHO4_ch12.pkl +gJS_sFM_cAll_d01_mJS1_ch02.pkl +gHO_sFM_cAll_d21_mHO5_ch20.pkl +gKR_sFM_cAll_d28_mKR5_ch06.pkl +gHO_sFM_cAll_d19_mHO5_ch06.pkl +gBR_sFM_cAll_d06_mBR2_ch16.pkl +gKR_sFM_cAll_d30_mKR3_ch21.pkl +gLH_sFM_cAll_d16_mLH5_ch06.pkl +gJS_sFM_cAll_d02_mJS1_ch11.pkl +gHO_sFM_cAll_d21_mHO0_ch15.pkl +gJS_sFM_cAll_d03_mJS5_ch14.pkl +gJB_sFM_cAll_d09_mJB3_ch18.pkl +gJS_sFM_cAll_d01_mJS0_ch01.pkl +gBR_sFM_cAll_d05_mBR3_ch10.pkl +gPO_sFM_cAll_d11_mPO2_ch14.pkl +gKR_sFM_cAll_d28_mKR4_ch05.pkl +gHO_sFM_cAll_d19_mHO4_ch05.pkl +gKR_sFM_cAll_d28_mKR3_ch07.pkl +gBR_sFM_cAll_d05_mBR5_ch12.pkl +gPO_sFM_cAll_d11_mPO0_ch08.pkl +gHO_sFM_cAll_d19_mHO2_ch07.pkl +gBR_sFM_cAll_d04_mBR1_ch02.pkl +gHO_sFM_cAll_d21_mHO3_ch21.pkl +gJB_sFM_cAll_d08_mJB5_ch14.pkl +gLH_sFM_cAll_d16_mLH3_ch04.pkl +gJS_sFM_cAll_d03_mJS2_ch03.pkl +gKR_sFM_cAll_d30_mKR0_ch15.pkl +gPO_sFM_cAll_d11_mPO1_ch09.pkl +gBR_sFM_cAll_d05_mBR4_ch13.pkl +gKR_sFM_cAll_d28_mKR3_ch04.pkl +gBR_sFM_cAll_d05_mBR4_ch11.pkl +gBR_sFM_cAll_d04_mBR0_ch01.pkl +gHO_sFM_cAll_d19_mHO3_ch04.pkl +gMH_sFM_cAll_d23_mMH0_ch08.pkl +gJB_sFM_cAll_d08_mJB1_ch09.pkl +gJB_sFM_cAll_d09_mJB4_ch19.pkl +gHO_sFM_cAll_d20_mHO2_ch10.pkl +gLH_sFM_cAll_d16_mLH3_ch07.pkl +gHO_sFM_cAll_d20_mHO3_ch11.pkl +gJB_sFM_cAll_d08_mJB0_ch08.pkl +gMH_sFM_cAll_d23_mMH1_ch09.pkl +gMH_sFM_cAll_d22_mMH1_ch07.pkl +gKR_sFM_cAll_d30_mKR1_ch16.pkl +gKR_sFM_cAll_d30_mKR5_ch20.pkl +gBR_sFM_cAll_d05_mBR5_ch14.pkl +gJB_sFM_cAll_d09_mJB1_ch16.pkl +gMH_sFM_cAll_d23_mMH5_ch13.pkl +gJB_sFM_cAll_d08_mJB4_ch12.pkl +gJB_sFM_cAll_d09_mJB5_ch20.pkl +gLH_sFM_cAll_d16_mLH2_ch03.pkl +gJS_sFM_cAll_d03_mJS3_ch04.pkl +gHO_sFM_cAll_d20_mHO3_ch14.pkl +gJB_sFM_cAll_d08_mJB5_ch13.pkl +gMH_sFM_cAll_d23_mMH4_ch12.pkl +gWA_sFM_cAll_d27_mWA2_ch17.pkl +gMH_sFM_cAll_d22_mMH1_ch02.pkl +gPO_sFM_cAll_d10_mPO0_ch01.pkl +gKR_sFM_cAll_d30_mKR4_ch19.pkl +gLO_sFM_cAll_d13_mLO1_ch02.pkl +gPO_sFM_cAll_d11_mPO5_ch13.pkl +gJB_sFM_cAll_d09_mJB0_ch15.pkl +gHO_sFM_cAll_d21_mHO3_ch18.pkl +gLH_sFM_cAll_d18_mLH0_ch21.pkl +gMH_sFM_cAll_d22_mMH0_ch01.pkl +gHO_sFM_cAll_d19_mHO2_ch03.pkl +gPO_sFM_cAll_d10_mPO1_ch02.pkl +gBR_sFM_cAll_d05_mBR1_ch08.pkl +gPO_sFM_cAll_d11_mPO4_ch12.pkl +gLO_sFM_cAll_d13_mLO0_ch01.pkl +gKR_sFM_cAll_d28_mKR2_ch03.pkl +gPO_sFM_cAll_d10_mPO4_ch05.pkl +gLO_sFM_cAll_d13_mLO5_ch06.pkl +gJB_sFM_cAll_d07_mJB3_ch04.pkl +gLO_sFM_cAll_d14_mLO0_ch08.pkl +gMH_sFM_cAll_d22_mMH5_ch06.pkl +gLO_sFM_cAll_d14_mLO1_ch09.pkl +gJS_sFM_cAll_d02_mJS3_ch04.pkl +gWA_sFM_cAll_d27_mWA3_ch18.pkl +gLO_sFM_cAll_d15_mLO4_ch19.pkl +gLO_sFM_cAll_d13_mLO4_ch07.pkl +gBR_sFM_cAll_d06_mBR1_ch15.pkl +gMH_sFM_cAll_d24_mMH4_ch19.pkl +gPO_sFM_cAll_d10_mPO5_ch06.pkl +gPO_sFM_cAll_d12_mPO4_ch19.pkl +gBR_sFM_cAll_d06_mBR5_ch21.pkl +gLO_sFM_cAll_d13_mLO4_ch05.pkl +gJB_sFM_cAll_d07_mJB3_ch07.pkl +gJS_sFM_cAll_d01_mJS2_ch03.pkl +gHO_sFM_cAll_d21_mHO2_ch17.pkl +gMH_sFM_cAll_d22_mMH4_ch05.pkl +gWA_sFM_cAll_d26_mWA3_ch14.pkl +gMH_sFM_cAll_d23_mMH0_ch14.pkl +gLH_sFM_cAll_d18_mLH4_ch19.pkl +gLO_sFM_cAll_d14_mLO5_ch14.pkl +gWA_sFM_cAll_d25_mWA3_ch04.pkl +gBR_sFM_cAll_d06_mBR4_ch20.pkl +gBR_sFM_cAll_d04_mBR4_ch07.pkl +gPO_sFM_cAll_d12_mPO1_ch16.pkl +gBR_sFM_cAll_d06_mBR4_ch18.pkl +gPO_sFM_cAll_d12_mPO5_ch20.pkl +gJS_sFM_cAll_d02_mJS2_ch03.pkl +gLH_sFM_cAll_d17_mLH2_ch10.pkl +gWA_sFM_cAll_d26_mWA3_ch11.pkl +gWA_sFM_cAll_d26_mWA2_ch10.pkl +gLH_sFM_cAll_d17_mLH3_ch11.pkl +gLO_sFM_cAll_d15_mLO0_ch15.pkl +gLH_sFM_cAll_d18_mLH1_ch16.pkl +gLH_sFM_cAll_d18_mLH5_ch20.pkl +gJB_sFM_cAll_d07_mJB2_ch03.pkl +gPO_sFM_cAll_d12_mPO5_ch21.pkl +gBR_sFM_cAll_d06_mBR5_ch19.pkl +gMH_sFM_cAll_d24_mMH0_ch15.pkl +gBR_sFM_cAll_d04_mBR5_ch06.pkl +gKR_sFM_cAll_d29_mKR3_ch11.pkl +gPO_sFM_cAll_d12_mPO0_ch15.pkl +gLO_sFM_cAll_d14_mLO5_ch13.pkl +gWA_sFM_cAll_d25_mWA2_ch03.pkl +gLO_sFM_cAll_d15_mLO4_ch21.pkl +gLO_sFM_cAll_d15_mLO1_ch16.pkl +gLH_sFM_cAll_d18_mLH0_ch15.pkl +gWA_sFM_cAll_d27_mWA2_ch21.pkl +gLO_sFM_cAll_d15_mLO5_ch20.pkl +gJS_sFM_cAll_d01_mJS3_ch04.pkl +gJB_sFM_cAll_d09_mJB1_ch21.pkl +gLO_sFM_cAll_d14_mLO4_ch12.pkl +gMH_sFM_cAll_d24_mMH1_ch16.pkl +gKR_sFM_cAll_d29_mKR2_ch10.pkl +gMH_sFM_cAll_d24_mMH5_ch20.pkl +gBR_sFM_cAll_d04_mBR4_ch05.pkl +gKR_sFM_cAll_d28_mKR1_ch02.pkl +gPO_sFM_cAll_d10_mPO2_ch03.pkl +gHO_sFM_cAll_d19_mHO1_ch02.pkl +gBR_sFM_cAll_d05_mBR2_ch09.pkl +gKR_sFM_cAll_d29_mKR4_ch12.pkl +gLO_sFM_cAll_d14_mLO2_ch10.pkl +gWA_sFM_cAll_d27_mWA0_ch15.pkl +gJS_sFM_cAll_d01_mJS5_ch06.pkl +gJS_sFM_cAll_d02_mJS0_ch08.pkl +gLH_sFM_cAll_d16_mLH0_ch01.pkl +gLO_sFM_cAll_d14_mLO3_ch11.pkl +gKR_sFM_cAll_d29_mKR5_ch13.pkl +gKR_sFM_cAll_d28_mKR0_ch01.pkl +gLO_sFM_cAll_d13_mLO2_ch03.pkl +gBR_sFM_cAll_d04_mBR3_ch04.pkl +gHO_sFM_cAll_d19_mHO0_ch01.pkl +gMH_sFM_cAll_d24_mMH2_ch21.pkl +gWA_sFM_cAll_d27_mWA5_ch20.pkl +gJB_sFM_cAll_d07_mJB0_ch23.pkl +gLH_sFM_cAll_d17_mLH5_ch13.pkl +gMH_sFM_cAll_d22_mMH2_ch03.pkl +gWA_sFM_cAll_d26_mWA4_ch12.pkl +gJS_sFM_cAll_d01_mJS4_ch05.pkl +gWA_sFM_cAll_d27_mWA1_ch16.pkl +gLH_sFM_cAll_d16_mLH1_ch02.pkl +gWA_sFM_cAll_d26_mWA5_ch13.pkl +gJB_sFM_cAll_d09_mJB2_ch17.pkl +gLH_sFM_cAll_d17_mLH4_ch12.pkl +gKR_sFM_cAll_d30_mKR2_ch17.pkl +gMH_sFM_cAll_d24_mMH3_ch18.pkl +gKR_sFM_cAll_d29_mKR5_ch14.pkl +gWA_sFM_cAll_d27_mWA4_ch19.pkl +gWA_sFM_cAll_d25_mWA5_ch06.pkl +gJS_sFM_cAll_d03_mJS0_ch01.pkl +gWA_sFM_cAll_d26_mWA1_ch09.pkl +gLO_sFM_cAll_d15_mLO3_ch18.pkl +gLH_sFM_cAll_d17_mLH0_ch08.pkl +gJB_sFM_cAll_d07_mJB4_ch05.pkl +gLH_sFM_cAll_d17_mLH1_ch09.pkl +gWA_sFM_cAll_d26_mWA0_ch08.pkl +gWA_sFM_cAll_d25_mWA5_ch07.pkl +gJS_sFM_cAll_d02_mJS5_ch10.pkl +gPO_sFM_cAll_d10_mPO3_ch04.pkl +gKR_sFM_cAll_d29_mKR1_ch09.pkl +gWA_sFM_cAll_d25_mWA4_ch05.pkl +gJS_sFM_cAll_d03_mJS1_ch02.pkl +gLH_sFM_cAll_d18_mLH3_ch18.pkl +gJB_sFM_cAll_d07_mJB5_ch06.pkl +gMH_sFM_cAll_d22_mMH3_ch04.pkl +gPO_sFM_cAll_d10_mPO3_ch07.pkl +gBR_sFM_cAll_d04_mBR2_ch03.pkl +gLO_sFM_cAll_d13_mLO3_ch04.pkl +gKR_sFM_cAll_d29_mKR0_ch08.pkl +gPO_sFM_cAll_d12_mPO3_ch18.pkl From 0f279c5202094064f97f2ab55f6f6f66ed86bf7f Mon Sep 17 00:00:00 2001 From: kelly anne Date: Mon, 10 Jun 2024 11:24:34 +0200 Subject: [PATCH 08/14] update perc cut --- koregraph/api/machine_learning/load_dataset.py | 6 +++--- koregraph/api/machine_learning/neural_network.py | 1 + .../machine_learning/train_workflow/next_chunks_workflow.py | 6 +++--- koregraph/params.py | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/koregraph/api/machine_learning/load_dataset.py b/koregraph/api/machine_learning/load_dataset.py index 2ed43c8..0c7e997 100644 --- a/koregraph/api/machine_learning/load_dataset.py +++ b/koregraph/api/machine_learning/load_dataset.py @@ -99,7 +99,7 @@ def load_chunk_preprocess_dataset() -> Tuple[ndarray, ndarray]: def load_next_chunks_preprocess_dataset(perc_cut: float = PERCENTAGE_CUT): - chore_names = ALL_ADVANCED_MOVE_NAMES[:100] + chore_names = ALL_ADVANCED_MOVE_NAMES[:10] X = None y = None for chore_name in chore_names: @@ -143,8 +143,8 @@ def load_next_chunks_preprocess_dataset(perc_cut: float = PERCENTAGE_CUT): else: X = append(X, X_tmp, axis=0) - X = X.reshape(-1, int(CHUNK_SIZE * (1 - perc_cut)) * 60, X.shape[-1]) - y = y.reshape(-1, int(CHUNK_SIZE * perc_cut) * 60 * 34) + X = X.reshape(-1, int((CHUNK_SIZE * (1 - perc_cut)) * 60), 17, 2) + y = y.reshape(-1, int(CHUNK_SIZE * perc_cut * 60 * 34)) print("X final shape", X.shape) print("y final shape", y.shape) diff --git a/koregraph/api/machine_learning/neural_network.py b/koregraph/api/machine_learning/neural_network.py index 6ddddcd..5c81083 100644 --- a/koregraph/api/machine_learning/neural_network.py +++ b/koregraph/api/machine_learning/neural_network.py @@ -140,6 +140,7 @@ def initialize_model_next_chunks(X, y) -> Model: # Conv2D(128, kernel_size=(3, 3), activation="relu", padding="same"), BatchNormalization(), Flatten(), + # lstm 512 Dense(256, activation="relu"), Dense(128, activation="relu"), Dropout(rate=0.2), diff --git a/koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py b/koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py index d24eae3..92fd6be 100644 --- a/koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py +++ b/koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py @@ -14,9 +14,9 @@ def train_workflow(model_name: str = "model"): - # X, y = load_preprocess_dataset() - X = load_pickle_object(GENERATED_FEATURES_DIRECTORY / "x.pkl") - y = load_pickle_object(GENERATED_FEATURES_DIRECTORY / "y.pkl") + X, y = load_preprocess_dataset() + # X = load_pickle_object(GENERATED_FEATURES_DIRECTORY / "x.pkl") + # y = load_pickle_object(GENERATED_FEATURES_DIRECTORY / "y.pkl") y = y.astype(float32) print("y has nan", isnan(y).any()) diff --git a/koregraph/params.py b/koregraph/params.py index c0a2789..032a755 100644 --- a/koregraph/params.py +++ b/koregraph/params.py @@ -99,7 +99,7 @@ X_MIN = -80 X_MAX = 0 -PERCENTAGE_CUT = 0.2 +PERCENTAGE_CUT = 0.05 GENERATED_FEATURES_DIRECTORY: Path = PROJECT_ROOT.joinpath( environ.get( From ccd89f068e70a801006ccdabbdc6aa6be50ddebe Mon Sep 17 00:00:00 2001 From: kelly anne Date: Mon, 10 Jun 2024 12:07:17 +0200 Subject: [PATCH 09/14] complexify conv2d, conv2d, lstm, lstm --- .../api/machine_learning/neural_network.py | 32 ++++++++++++++++--- .../train_workflow/next_chunks_workflow.py | 4 ++- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/koregraph/api/machine_learning/neural_network.py b/koregraph/api/machine_learning/neural_network.py index 5c81083..fefa9c4 100644 --- a/koregraph/api/machine_learning/neural_network.py +++ b/koregraph/api/machine_learning/neural_network.py @@ -13,6 +13,8 @@ Conv2D, BatchNormalization, Flatten, + Reshape, + TimeDistributed, Input, ) from keras.initializers import glorot_uniform @@ -128,19 +130,39 @@ def initialize_model_next_chunks(X, y) -> Model: print("x 0 shape", X[0].shape) new_model = Sequential( [ - # Input(), + Input(X[0].shape), Conv2D( 258, kernel_size=(3, 3), activation="relu", - input_shape=X[0].shape, + # input_shape=X[0].shape, padding="same", ), - # Dropout(rate=0.2), - # Conv2D(128, kernel_size=(3, 3), activation="relu", padding="same"), + Dropout(rate=0.2), + Conv2D(128, kernel_size=(3, 3), activation="relu", padding="same"), BatchNormalization(), - Flatten(), + # Flatten(), # lstm 512 + # Reshape((285*17*2, )), + + # Reshape((-1, 16*510)), + TimeDistributed(Flatten()), + Bidirectional( + LSTM( + 512, + activation="tanh", + kernel_initializer=glorot_uniform(), + return_sequences=True, + ), + ), + Bidirectional( + LSTM( + 256, + activation="tanh", + kernel_initializer=glorot_uniform(), + return_sequences=False, + ), + ), Dense(256, activation="relu"), Dense(128, activation="relu"), Dropout(rate=0.2), diff --git a/koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py b/koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py index 92fd6be..9479604 100644 --- a/koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py +++ b/koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py @@ -34,6 +34,8 @@ def train_workflow(model_name: str = "model"): model = initialize_model_next_chunks(X, y) + model.summary() + history = model.fit( x=X, y=y, @@ -52,7 +54,7 @@ def train_workflow(model_name: str = "model"): initial_value_threshold=None, ), EarlyStopping( - monitor="val_loss", patience=7, verbose=0, restore_best_weights=True + monitor="val_loss", patience=50, verbose=0, restore_best_weights=True ), ], ) From b2c22efee841d3a2bc03e5567acbd888647cde25 Mon Sep 17 00:00:00 2001 From: kelly anne Date: Tue, 11 Jun 2024 09:32:08 +0200 Subject: [PATCH 10/14] working chunks with rolling prediction --- koregraph/api/machine_learning/loss.py | 30 +++++++++ .../api/machine_learning/neural_network.py | 62 ++++++++++++++----- .../machine_learning/prediction_workflow.py | 55 +++++++++++++--- .../train_workflow/next_chunks_workflow.py | 2 +- 4 files changed, 123 insertions(+), 26 deletions(-) create mode 100644 koregraph/api/machine_learning/loss.py diff --git a/koregraph/api/machine_learning/loss.py b/koregraph/api/machine_learning/loss.py new file mode 100644 index 0000000..8633a0d --- /dev/null +++ b/koregraph/api/machine_learning/loss.py @@ -0,0 +1,30 @@ +from tensorflow import reduce_mean, square, expand_dims +from keras.saving import register_keras_serializable + + +def distance_frame_to_frame(frame1, frame2): + distance = 0 + for i in range(0, 34, 2): + distance += (frame1[:, i] - frame2[:, i]) ** 2 + ( + frame1[:, (i + 1)] - frame2[:, (i + 1)] + ) ** 2 + + return distance + + +@register_keras_serializable() +def my_mse(y_true, y_pred): + distances = distance_frame_to_frame(y_true[::, :], y_pred[::, :]) + + return reduce_mean(square(distances)) + + +@register_keras_serializable() +def my_mse_maximise_movement(y_true, y_pred): + first_frame = expand_dims(y_pred[0, :], 0) + last_frame = expand_dims(y_pred[-1, :], 0) + distances = distance_frame_to_frame( + y_true[::, :], y_pred[::, :] + ) - distance_frame_to_frame(first_frame, last_frame) + + return reduce_mean(square(distances)) diff --git a/koregraph/api/machine_learning/neural_network.py b/koregraph/api/machine_learning/neural_network.py index fefa9c4..49bb6f5 100644 --- a/koregraph/api/machine_learning/neural_network.py +++ b/koregraph/api/machine_learning/neural_network.py @@ -11,9 +11,9 @@ Dropout, Bidirectional, Conv2D, - BatchNormalization, + Conv2DTranspose, + MaxPooling2D, Flatten, - Reshape, TimeDistributed, Input, ) @@ -21,6 +21,8 @@ from keras.models import Sequential, Model from keras.optimizers import Adam +from koregraph.api.machine_learning.loss import my_mse + def prepare_model(X, y) -> Model: """Initilizer a non compiled model with layers within. @@ -125,27 +127,53 @@ def initialize_model_next_chunks(X, y) -> Model: Model: The compiled model. """ - # normalization_layer = Normalization() - # normalization_layer.adapt(X) - print("x 0 shape", X[0].shape) new_model = Sequential( [ Input(X[0].shape), Conv2D( - 258, + 512, kernel_size=(3, 3), activation="relu", - # input_shape=X[0].shape, padding="same", ), - Dropout(rate=0.2), - Conv2D(128, kernel_size=(3, 3), activation="relu", padding="same"), - BatchNormalization(), - # Flatten(), - # lstm 512 - # Reshape((285*17*2, )), - - # Reshape((-1, 16*510)), + MaxPooling2D((2, 2), padding="same"), + Conv2D( + 256, + kernel_size=(3, 3), + activation="relu", + padding="same", + ), + MaxPooling2D((2, 2), padding="same"), + Conv2D( + 128, + kernel_size=(3, 3), + activation="relu", + padding="same", + ), + MaxPooling2D((2, 2), padding="same"), + Conv2D( + 64, + kernel_size=(3, 3), + activation="relu", + padding="same", + ), + MaxPooling2D((2, 2), padding="same"), + Conv2D( + 128, + kernel_size=(3, 3), + activation="relu", + padding="same", + ), + MaxPooling2D((2, 2), padding="same"), + Conv2DTranspose( + 32, (3, 3), strides=(2, 2), padding="same", activation="relu" + ), + Conv2DTranspose( + 16, (3, 3), strides=(2, 2), padding="same", activation="relu" + ), + Conv2DTranspose( + 1, (3, 3), strides=(2, 2), padding="same", activation="relu" + ), TimeDistributed(Flatten()), Bidirectional( LSTM( @@ -166,13 +194,13 @@ def initialize_model_next_chunks(X, y) -> Model: Dense(256, activation="relu"), Dense(128, activation="relu"), Dropout(rate=0.2), - Dense(64, activation="relu"), # , activity_regularizer="l2"), + Dense(64, activation="relu"), Dropout(rate=0.2), Dense(y.shape[1], activation="relu"), ] ) - new_model.compile(loss="mse", optimizer="adam", metrics=["mae"]) + new_model.compile(loss=my_mse, optimizer="adam", metrics=["mae"]) return new_model diff --git a/koregraph/api/machine_learning/prediction_workflow.py b/koregraph/api/machine_learning/prediction_workflow.py index 7c6ab0c..47b8eb1 100644 --- a/koregraph/api/machine_learning/prediction_workflow.py +++ b/koregraph/api/machine_learning/prediction_workflow.py @@ -3,7 +3,7 @@ from koregraph.managers.choregraphy import save_choregaphy_chunk from koregraph.utils.pickle import load_pickle_object from koregraph.api.music_to_numpy import music_to_numpy -from koregraph.api.posture_proc import upscale_posture_pred +from koregraph.api.posture_proc import upscale_posture_pred, scale_posture_pred from koregraph.tools.video_builder import ( keypoints_video_audio_builder_from_choreography, ) @@ -68,6 +68,20 @@ def predict_next_move( chunk_id: int = 0, perc_cut: float = PERCENTAGE_CUT, ): + def build_next_input(first_frames, prediction): + prediction_frame_size = int((CHUNK_SIZE * perc_cut) * 60) + first_frames = first_frames.reshape( + -1, int((CHUNK_SIZE * (1 - perc_cut)) * 60), 17, 2 + ) + prediction = prediction.reshape(-1, prediction_frame_size, 17, 2) + + print(first_frames.shape) + print(prediction.shape) + return concatenate( + (first_frames[0, prediction_frame_size:, :, :], prediction[0, :, :, :]), + axis=0, + ) + model_path = MODEL_OUTPUT_DIRECTORY / (model_name + ".pkl") model = load_pickle_object(model_path) @@ -93,30 +107,50 @@ def predict_next_move( / (f"{chore_chunk_name}_{chunk_id}.pkl") ) input = chore["keypoints2d"] - input[:, :, 0] = input[:, :, 0] / FRAME_FORMAT[0] - input[:, :, 1] = input[:, :, 1] / FRAME_FORMAT[1] + + input = scale_posture_pred(input) input = input.reshape(-1, 34) input, _ = cut_percentage(input.reshape(-1, 34), perc_cut) - # chore, _ = cut_percentage(chore.reshape(-1, 34), perc_cut) print("min input: ", input.min()) print("max input: ", input.max()) - # input = concatenate((chore, audio), axis=1) - input = input.reshape(1, 240, 17, 2) - # input = input.reshape(-1, 8160) + input = input.reshape(-1, int((CHUNK_SIZE * (1 - perc_cut)) * 60), 17, 2) + print(input.shape) - # print(input) + all_output = input + + # Predict first frame + print("Prediction 0") prediction = model.predict(input) print("prediction shape", prediction.shape) print("prediction min", prediction.min()) print("prediction max", prediction.max()) + all_output = concatenate( + (all_output[0].reshape(-1, 17, 2), prediction.reshape(-1, 17, 2)), axis=0 + ) + + # Predict next frame + for i in range(4): + print(f"Prediction {i+1}") + input = build_next_input(input, prediction) + input = input.reshape(-1, int((CHUNK_SIZE * (1 - perc_cut)) * 60), 17, 2) + print(f"Next input shape {input.shape}") + prediction = model.predict(input) + all_output = concatenate((all_output, prediction.reshape(-1, 17, 2)), axis=0) + print("prediction shape", prediction.shape) + print("prediction min", prediction.min()) + print("prediction max", prediction.max()) + prediction = upscale_posture_pred(prediction) + all_output = upscale_posture_pred(all_output) print("Prediction shape:", prediction.shape) + print("All shape:", all_output.shape) prediction_name = ( model_name.replace("_", "-") + "_sBM_cAll_d00_" + audio_name + "_ch01" ) + output_name = model_name.replace("_", "-") + "_sBM_cAll_d00_" + audio_name + "_ch02" # print('chore') # print(chore.shape) @@ -126,11 +160,16 @@ def predict_next_move( chore = Choregraphy( prediction_name, output.reshape(-1, 17, 2), np_ones(output.shape[0]) ) + chore_all = Choregraphy( + output_name, all_output.reshape(-1, 17, 2), np_ones(all_output.shape[0]) + ) # Save prediction to pkl save_choregaphy_chunk(chore, PREDICTION_OUTPUT_DIRECTORY) + save_choregaphy_chunk(chore_all, PREDICTION_OUTPUT_DIRECTORY) # Create video keypoints_video_audio_builder_from_choreography(chore) + # keypoints_video_audio_builder_from_choreography(chore_all) print("Happy viewing!") diff --git a/koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py b/koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py index 9479604..bf0d27f 100644 --- a/koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py +++ b/koregraph/api/machine_learning/train_workflow/next_chunks_workflow.py @@ -41,7 +41,7 @@ def train_workflow(model_name: str = "model"): y=y, validation_split=0.2, batch_size=16, - epochs=1000, + epochs=50, callbacks=[ ModelCheckpoint( WEIGHTS_BACKUP_DIRECTORY / f"{model_name}_backup.keras", From 2820daa9347108eff82993ac3fb5341465bb6dcd Mon Sep 17 00:00:00 2001 From: kelly anne Date: Tue, 11 Jun 2024 12:31:46 +0200 Subject: [PATCH 11/14] save model to folder as keras --- .../api/machine_learning/load_dataset.py | 7 ++- .../machine_learning/prediction_workflow.py | 12 +++- .../preprocessing/posture_preprocessing.py | 2 + .../api/training/next_chunks_workflow.py | 55 +++++++++++++++---- koregraph/cli/model.py | 4 ++ koregraph/cli/predict.py | 1 + 6 files changed, 65 insertions(+), 16 deletions(-) diff --git a/koregraph/api/machine_learning/load_dataset.py b/koregraph/api/machine_learning/load_dataset.py index 7ac5ace..d673005 100644 --- a/koregraph/api/machine_learning/load_dataset.py +++ b/koregraph/api/machine_learning/load_dataset.py @@ -10,7 +10,10 @@ from koregraph.utils.controllers.pickles import load_pickle_object, save_object_pickle from koregraph.api.preprocessing.audio_preprocessing import music_to_numpy -from koregraph.api.preprocessing.posture_preprocessing import fill_forward, cut_percentage +from koregraph.api.preprocessing.posture_preprocessing import ( + fill_forward, + cut_percentage, +) from koregraph.config.params import ( GENERATED_KEYPOINTS_DIRECTORY, GENERATED_AUDIO_DIRECTORY, @@ -116,7 +119,7 @@ def load_next_chunks_preprocess_dataset( for file in listdir(chore_path): chunk_id = file.replace(".pkl", "").split("_")[-1] chore_filepath = chore_path / file - music_filepath = music_path / f"{music_name}_{chunk_id}.mp3" + # music_filepath = music_path / f"{music_name}_{chunk_id}.mp3" # audio_tmp = music_to_numpy(music_filepath) chore_tmp = load_pickle_object(chore_filepath)["keypoints2d"] diff --git a/koregraph/api/machine_learning/prediction_workflow.py b/koregraph/api/machine_learning/prediction_workflow.py index 8d317ea..cfe2940 100644 --- a/koregraph/api/machine_learning/prediction_workflow.py +++ b/koregraph/api/machine_learning/prediction_workflow.py @@ -78,6 +78,7 @@ def predict_next_move( chore_chunk_name: str = "gBR_sFM_cAll_d04_mBR0_ch01", chunk_id: int = 0, perc_cut: float = PERCENTAGE_CUT, + backup: bool = False, ): def build_next_input(first_frames, prediction): prediction_frame_size = int((CHUNK_SIZE * perc_cut) * 60) @@ -93,8 +94,15 @@ def build_next_input(first_frames, prediction): axis=0, ) - model_path = MODEL_OUTPUT_DIRECTORY / (model_name + ".pkl") - model = load_pickle_object(model_path) + # model_path = MODEL_OUTPUT_DIRECTORY / (model_name + ".pkl") + # model = load_pickle_object(model_path) + + model_path = ( + MODEL_OUTPUT_DIRECTORY + / model_name + / f"{model_name}{'_backup' if backup else ''}.keras" + ) + model = load_model(model_path) audio_filepath = ( GENERATED_AUDIO_DIRECTORY diff --git a/koregraph/api/preprocessing/posture_preprocessing.py b/koregraph/api/preprocessing/posture_preprocessing.py index 17824f6..fa3efe1 100644 --- a/koregraph/api/preprocessing/posture_preprocessing.py +++ b/koregraph/api/preprocessing/posture_preprocessing.py @@ -192,10 +192,12 @@ def fill_forward(arr): return arr + def cut_percentage(x: ndarray, perc: float) -> Tuple[ndarray, ndarray]: idx = len(x) - int(len(x) * perc) return x[:idx], x[idx:] + if __name__ == "__main__": export_choregraphy_keypoints( Choregraphy( diff --git a/koregraph/api/training/next_chunks_workflow.py b/koregraph/api/training/next_chunks_workflow.py index 4d5eb05..dd7c853 100644 --- a/koregraph/api/training/next_chunks_workflow.py +++ b/koregraph/api/training/next_chunks_workflow.py @@ -12,14 +12,20 @@ CHUNK_SIZE, PERCENTAGE_CUT, WEIGHTS_BACKUP_DIRECTORY, + MODEL_OUTPUT_DIRECTORY, ) +from koregraph.api.machine_learning.callbacks import GCSCallback, HistorySaver def train_workflow( model_name: str = "model", + epochs: int = 16, + batch_size: int = 16, dataset_size: float = 1.0, backup_model: Model = None, initial_epoch: int = 0, + patience: int = 20, + with_cloud: bool = False, ): X, y = load_preprocess_dataset(dataset_size=dataset_size) @@ -42,16 +48,11 @@ def train_workflow( model = initialize_model_next_chunks(X, y) if backup_model is None else backup_model - model.summary() + model_backup_path = MODEL_OUTPUT_DIRECTORY / model_name + model_backup_path.mkdir(parents=True, exist_ok=True) - history = model.fit( - x=X, - y=y, - validation_split=0.2, - batch_size=16, - epochs=10, - initial_epoch=initial_epoch, - callbacks=[ + model_callbacks = ( + [ ModelCheckpoint( WEIGHTS_BACKUP_DIRECTORY / f"{model_name}_backup.keras", monitor="val_loss", @@ -63,13 +64,43 @@ def train_workflow( initial_value_threshold=None, ), EarlyStopping( - monitor="val_loss", patience=50, verbose=0, restore_best_weights=True + monitor="val_loss", + patience=patience, + verbose=0, + restore_best_weights=True, ), ], ) + model.summary() + + if with_cloud: + model_callbacks.append(GCSCallback(model_backup_path, "koregraph")) + + history = model.fit( + x=X, + y=y, + validation_split=0.2, + batch_size=batch_size, + epochs=epochs, + initial_epoch=initial_epoch, + callbacks=model_callbacks, + ) + + print("Exporting model locally") + (MODEL_OUTPUT_DIRECTORY / model_name).mkdir(exist_ok=True, parents=True) + model.save(MODEL_OUTPUT_DIRECTORY / model_name / f"{model_name}.keras") + save_object_pickle( + history, + model_name + "_history", + MODEL_OUTPUT_DIRECTORY / model_name / f"{model_name}_history.pkl", + ) - save_object_pickle(model, model_name) - save_object_pickle(history, model_name + "_history") + if with_cloud: + print("Exporting model to google cloud storage") + GCSCallback(model_backup_path, "koregraph").upload_file_to_gcs( + MODEL_OUTPUT_DIRECTORY / model_name / f"{model_name}.keras", + f"generated/models/{model_name}/", + ) if __name__ == "__main__": diff --git a/koregraph/cli/model.py b/koregraph/cli/model.py index 1ca4dea..8ec6879 100644 --- a/koregraph/cli/model.py +++ b/koregraph/cli/model.py @@ -145,9 +145,13 @@ def main(): print("Training with chunks: predicting next X seconds") train_pred_next_workflow( model_name=model_name, + epochs=epochs, + batch_size=batch_size, dataset_size=dataset_size, backup_model=model, initial_epoch=initial_epoch, + patience=patience, + with_cloud=with_cloud, ) else: print("Running training locally") diff --git a/koregraph/cli/predict.py b/koregraph/cli/predict.py index e23abc8..afdafa4 100644 --- a/koregraph/cli/predict.py +++ b/koregraph/cli/predict.py @@ -82,6 +82,7 @@ def main(): model_name=model_name, chore_chunk_name=choregraphy, chunk_id=chunk_id, + backup=backup, ) else: predict_api( From 409fba919e8992042178187e7c7551cbbf088a05 Mon Sep 17 00:00:00 2001 From: kelly anne Date: Tue, 11 Jun 2024 12:57:48 +0200 Subject: [PATCH 12/14] Tests after merger --- koregraph/api/machine_learning/load_dataset.py | 4 ++-- koregraph/api/preprocessing/dataset_preprocessing.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/koregraph/api/machine_learning/load_dataset.py b/koregraph/api/machine_learning/load_dataset.py index d673005..bf5eb7e 100644 --- a/koregraph/api/machine_learning/load_dataset.py +++ b/koregraph/api/machine_learning/load_dataset.py @@ -110,10 +110,10 @@ def load_next_chunks_preprocess_dataset( y = None for chore in chore_names: chore_name = chore.name - music_name = chore.music + # music_name = chore.music chore_path = GENERATED_KEYPOINTS_DIRECTORY / chore_name / str(CHUNK_SIZE) - music_path = GENERATED_AUDIO_DIRECTORY / music_name / str(CHUNK_SIZE) + # music_path = GENERATED_AUDIO_DIRECTORY / music_name / str(CHUNK_SIZE) print(f"Parsing {chore_name} chunks") for file in listdir(chore_path): diff --git a/koregraph/api/preprocessing/dataset_preprocessing.py b/koregraph/api/preprocessing/dataset_preprocessing.py index dd103d0..57569ce 100644 --- a/koregraph/api/preprocessing/dataset_preprocessing.py +++ b/koregraph/api/preprocessing/dataset_preprocessing.py @@ -90,7 +90,10 @@ def generate_training_pickles( # Ensure X and y have the same length if len(train_choregraphy) != len(train_audio): - train_audio = train_audio[: len(train_choregraphy)] + if len(train_choregraphy) > len(train_audio): + train_choregraphy = train_choregraphy[: len(train_audio)] + else: + train_audio = train_audio[: len(train_choregraphy)] assert len(train_choregraphy) == len( train_audio From 3db81c16604f9e1cbd0f63edcca7a355a8d5cd06 Mon Sep 17 00:00:00 2001 From: Kelly-an <166104677+Kelly-an@users.noreply.github.com> Date: Tue, 11 Jun 2024 13:37:40 +0200 Subject: [PATCH 13/14] Delete notebook/all_advanced_move_names.csv --- notebook/all_advanced_move_names.csv | 212 --------------------------- 1 file changed, 212 deletions(-) delete mode 100644 notebook/all_advanced_move_names.csv diff --git a/notebook/all_advanced_move_names.csv b/notebook/all_advanced_move_names.csv deleted file mode 100644 index 1e1aba9..0000000 --- a/notebook/all_advanced_move_names.csv +++ /dev/null @@ -1,212 +0,0 @@ -gKR_sFM_cAll_d30_mKR3_ch18.pkl -gMH_sFM_cAll_d24_mMH2_ch17.pkl -gPO_sFM_cAll_d11_mPO2_ch10.pkl -gJB_sFM_cAll_d07_mJB0_ch01.pkl -gLO_sFM_cAll_d15_mLO2_ch17.pkl -gWA_sFM_cAll_d25_mWA1_ch02.pkl -gPO_sFM_cAll_d11_mPO3_ch11.pkl -gJB_sFM_cAll_d07_mKK1_ch01.pkl -gJB_sFM_cAll_d07_mJB1_ch02.pkl -gMH_sFM_cAll_d23_mMH2_ch10.pkl -gJB_sFM_cAll_d08_mJB3_ch11.pkl -gJS_sFM_cAll_d02_mJS4_ch09.pkl -gJS_sFM_cAll_d03_mJS5_ch13.pkl -gHO_sFM_cAll_d20_mHO0_ch08.pkl -gLH_sFM_cAll_d18_mLH2_ch17.pkl -gJS_sFM_cAll_d02_mJS1_ch02.pkl -gHO_sFM_cAll_d20_mHO1_ch09.pkl -gJS_sFM_cAll_d03_mJS4_ch12.pkl -gWA_sFM_cAll_d25_mWA0_ch01.pkl -gJB_sFM_cAll_d08_mJB2_ch10.pkl -gHO_sFM_cAll_d21_mHO4_ch19.pkl -gJS_sFM_cAll_d01_mJS1_ch07.pkl -gMH_sFM_cAll_d23_mMH3_ch11.pkl -gPO_sFM_cAll_d12_mPO2_ch17.pkl -gBR_sFM_cAll_d06_mBR3_ch17.pkl -gHO_sFM_cAll_d20_mHO5_ch13.pkl -gLH_sFM_cAll_d16_mLH4_ch05.pkl -gLH_sFM_cAll_d17_mLH0_ch14.pkl -gHO_sFM_cAll_d21_mHO1_ch16.pkl -gHO_sFM_cAll_d20_mHO4_ch12.pkl -gJS_sFM_cAll_d01_mJS1_ch02.pkl -gHO_sFM_cAll_d21_mHO5_ch20.pkl -gKR_sFM_cAll_d28_mKR5_ch06.pkl -gHO_sFM_cAll_d19_mHO5_ch06.pkl -gBR_sFM_cAll_d06_mBR2_ch16.pkl -gKR_sFM_cAll_d30_mKR3_ch21.pkl -gLH_sFM_cAll_d16_mLH5_ch06.pkl -gJS_sFM_cAll_d02_mJS1_ch11.pkl -gHO_sFM_cAll_d21_mHO0_ch15.pkl -gJS_sFM_cAll_d03_mJS5_ch14.pkl -gJB_sFM_cAll_d09_mJB3_ch18.pkl -gJS_sFM_cAll_d01_mJS0_ch01.pkl -gBR_sFM_cAll_d05_mBR3_ch10.pkl -gPO_sFM_cAll_d11_mPO2_ch14.pkl -gKR_sFM_cAll_d28_mKR4_ch05.pkl -gHO_sFM_cAll_d19_mHO4_ch05.pkl -gKR_sFM_cAll_d28_mKR3_ch07.pkl -gBR_sFM_cAll_d05_mBR5_ch12.pkl -gPO_sFM_cAll_d11_mPO0_ch08.pkl -gHO_sFM_cAll_d19_mHO2_ch07.pkl -gBR_sFM_cAll_d04_mBR1_ch02.pkl -gHO_sFM_cAll_d21_mHO3_ch21.pkl -gJB_sFM_cAll_d08_mJB5_ch14.pkl -gLH_sFM_cAll_d16_mLH3_ch04.pkl -gJS_sFM_cAll_d03_mJS2_ch03.pkl -gKR_sFM_cAll_d30_mKR0_ch15.pkl -gPO_sFM_cAll_d11_mPO1_ch09.pkl -gBR_sFM_cAll_d05_mBR4_ch13.pkl -gKR_sFM_cAll_d28_mKR3_ch04.pkl -gBR_sFM_cAll_d05_mBR4_ch11.pkl -gBR_sFM_cAll_d04_mBR0_ch01.pkl -gHO_sFM_cAll_d19_mHO3_ch04.pkl -gMH_sFM_cAll_d23_mMH0_ch08.pkl -gJB_sFM_cAll_d08_mJB1_ch09.pkl -gJB_sFM_cAll_d09_mJB4_ch19.pkl -gHO_sFM_cAll_d20_mHO2_ch10.pkl -gLH_sFM_cAll_d16_mLH3_ch07.pkl -gHO_sFM_cAll_d20_mHO3_ch11.pkl -gJB_sFM_cAll_d08_mJB0_ch08.pkl -gMH_sFM_cAll_d23_mMH1_ch09.pkl -gMH_sFM_cAll_d22_mMH1_ch07.pkl -gKR_sFM_cAll_d30_mKR1_ch16.pkl -gKR_sFM_cAll_d30_mKR5_ch20.pkl -gBR_sFM_cAll_d05_mBR5_ch14.pkl -gJB_sFM_cAll_d09_mJB1_ch16.pkl -gMH_sFM_cAll_d23_mMH5_ch13.pkl -gJB_sFM_cAll_d08_mJB4_ch12.pkl -gJB_sFM_cAll_d09_mJB5_ch20.pkl -gLH_sFM_cAll_d16_mLH2_ch03.pkl -gJS_sFM_cAll_d03_mJS3_ch04.pkl -gHO_sFM_cAll_d20_mHO3_ch14.pkl -gJB_sFM_cAll_d08_mJB5_ch13.pkl -gMH_sFM_cAll_d23_mMH4_ch12.pkl -gWA_sFM_cAll_d27_mWA2_ch17.pkl -gMH_sFM_cAll_d22_mMH1_ch02.pkl -gPO_sFM_cAll_d10_mPO0_ch01.pkl -gKR_sFM_cAll_d30_mKR4_ch19.pkl -gLO_sFM_cAll_d13_mLO1_ch02.pkl -gPO_sFM_cAll_d11_mPO5_ch13.pkl -gJB_sFM_cAll_d09_mJB0_ch15.pkl -gHO_sFM_cAll_d21_mHO3_ch18.pkl -gLH_sFM_cAll_d18_mLH0_ch21.pkl -gMH_sFM_cAll_d22_mMH0_ch01.pkl -gHO_sFM_cAll_d19_mHO2_ch03.pkl -gPO_sFM_cAll_d10_mPO1_ch02.pkl -gBR_sFM_cAll_d05_mBR1_ch08.pkl -gPO_sFM_cAll_d11_mPO4_ch12.pkl -gLO_sFM_cAll_d13_mLO0_ch01.pkl -gKR_sFM_cAll_d28_mKR2_ch03.pkl -gPO_sFM_cAll_d10_mPO4_ch05.pkl -gLO_sFM_cAll_d13_mLO5_ch06.pkl -gJB_sFM_cAll_d07_mJB3_ch04.pkl -gLO_sFM_cAll_d14_mLO0_ch08.pkl -gMH_sFM_cAll_d22_mMH5_ch06.pkl -gLO_sFM_cAll_d14_mLO1_ch09.pkl -gJS_sFM_cAll_d02_mJS3_ch04.pkl -gWA_sFM_cAll_d27_mWA3_ch18.pkl -gLO_sFM_cAll_d15_mLO4_ch19.pkl -gLO_sFM_cAll_d13_mLO4_ch07.pkl -gBR_sFM_cAll_d06_mBR1_ch15.pkl -gMH_sFM_cAll_d24_mMH4_ch19.pkl -gPO_sFM_cAll_d10_mPO5_ch06.pkl -gPO_sFM_cAll_d12_mPO4_ch19.pkl -gBR_sFM_cAll_d06_mBR5_ch21.pkl -gLO_sFM_cAll_d13_mLO4_ch05.pkl -gJB_sFM_cAll_d07_mJB3_ch07.pkl -gJS_sFM_cAll_d01_mJS2_ch03.pkl -gHO_sFM_cAll_d21_mHO2_ch17.pkl -gMH_sFM_cAll_d22_mMH4_ch05.pkl -gWA_sFM_cAll_d26_mWA3_ch14.pkl -gMH_sFM_cAll_d23_mMH0_ch14.pkl -gLH_sFM_cAll_d18_mLH4_ch19.pkl -gLO_sFM_cAll_d14_mLO5_ch14.pkl -gWA_sFM_cAll_d25_mWA3_ch04.pkl -gBR_sFM_cAll_d06_mBR4_ch20.pkl -gBR_sFM_cAll_d04_mBR4_ch07.pkl -gPO_sFM_cAll_d12_mPO1_ch16.pkl -gBR_sFM_cAll_d06_mBR4_ch18.pkl -gPO_sFM_cAll_d12_mPO5_ch20.pkl -gJS_sFM_cAll_d02_mJS2_ch03.pkl -gLH_sFM_cAll_d17_mLH2_ch10.pkl -gWA_sFM_cAll_d26_mWA3_ch11.pkl -gWA_sFM_cAll_d26_mWA2_ch10.pkl -gLH_sFM_cAll_d17_mLH3_ch11.pkl -gLO_sFM_cAll_d15_mLO0_ch15.pkl -gLH_sFM_cAll_d18_mLH1_ch16.pkl -gLH_sFM_cAll_d18_mLH5_ch20.pkl -gJB_sFM_cAll_d07_mJB2_ch03.pkl -gPO_sFM_cAll_d12_mPO5_ch21.pkl -gBR_sFM_cAll_d06_mBR5_ch19.pkl -gMH_sFM_cAll_d24_mMH0_ch15.pkl -gBR_sFM_cAll_d04_mBR5_ch06.pkl -gKR_sFM_cAll_d29_mKR3_ch11.pkl -gPO_sFM_cAll_d12_mPO0_ch15.pkl -gLO_sFM_cAll_d14_mLO5_ch13.pkl -gWA_sFM_cAll_d25_mWA2_ch03.pkl -gLO_sFM_cAll_d15_mLO4_ch21.pkl -gLO_sFM_cAll_d15_mLO1_ch16.pkl -gLH_sFM_cAll_d18_mLH0_ch15.pkl -gWA_sFM_cAll_d27_mWA2_ch21.pkl -gLO_sFM_cAll_d15_mLO5_ch20.pkl -gJS_sFM_cAll_d01_mJS3_ch04.pkl -gJB_sFM_cAll_d09_mJB1_ch21.pkl -gLO_sFM_cAll_d14_mLO4_ch12.pkl -gMH_sFM_cAll_d24_mMH1_ch16.pkl -gKR_sFM_cAll_d29_mKR2_ch10.pkl -gMH_sFM_cAll_d24_mMH5_ch20.pkl -gBR_sFM_cAll_d04_mBR4_ch05.pkl -gKR_sFM_cAll_d28_mKR1_ch02.pkl -gPO_sFM_cAll_d10_mPO2_ch03.pkl -gHO_sFM_cAll_d19_mHO1_ch02.pkl -gBR_sFM_cAll_d05_mBR2_ch09.pkl -gKR_sFM_cAll_d29_mKR4_ch12.pkl -gLO_sFM_cAll_d14_mLO2_ch10.pkl -gWA_sFM_cAll_d27_mWA0_ch15.pkl -gJS_sFM_cAll_d01_mJS5_ch06.pkl -gJS_sFM_cAll_d02_mJS0_ch08.pkl -gLH_sFM_cAll_d16_mLH0_ch01.pkl -gLO_sFM_cAll_d14_mLO3_ch11.pkl -gKR_sFM_cAll_d29_mKR5_ch13.pkl -gKR_sFM_cAll_d28_mKR0_ch01.pkl -gLO_sFM_cAll_d13_mLO2_ch03.pkl -gBR_sFM_cAll_d04_mBR3_ch04.pkl -gHO_sFM_cAll_d19_mHO0_ch01.pkl -gMH_sFM_cAll_d24_mMH2_ch21.pkl -gWA_sFM_cAll_d27_mWA5_ch20.pkl -gJB_sFM_cAll_d07_mJB0_ch23.pkl -gLH_sFM_cAll_d17_mLH5_ch13.pkl -gMH_sFM_cAll_d22_mMH2_ch03.pkl -gWA_sFM_cAll_d26_mWA4_ch12.pkl -gJS_sFM_cAll_d01_mJS4_ch05.pkl -gWA_sFM_cAll_d27_mWA1_ch16.pkl -gLH_sFM_cAll_d16_mLH1_ch02.pkl -gWA_sFM_cAll_d26_mWA5_ch13.pkl -gJB_sFM_cAll_d09_mJB2_ch17.pkl -gLH_sFM_cAll_d17_mLH4_ch12.pkl -gKR_sFM_cAll_d30_mKR2_ch17.pkl -gMH_sFM_cAll_d24_mMH3_ch18.pkl -gKR_sFM_cAll_d29_mKR5_ch14.pkl -gWA_sFM_cAll_d27_mWA4_ch19.pkl -gWA_sFM_cAll_d25_mWA5_ch06.pkl -gJS_sFM_cAll_d03_mJS0_ch01.pkl -gWA_sFM_cAll_d26_mWA1_ch09.pkl -gLO_sFM_cAll_d15_mLO3_ch18.pkl -gLH_sFM_cAll_d17_mLH0_ch08.pkl -gJB_sFM_cAll_d07_mJB4_ch05.pkl -gLH_sFM_cAll_d17_mLH1_ch09.pkl -gWA_sFM_cAll_d26_mWA0_ch08.pkl -gWA_sFM_cAll_d25_mWA5_ch07.pkl -gJS_sFM_cAll_d02_mJS5_ch10.pkl -gPO_sFM_cAll_d10_mPO3_ch04.pkl -gKR_sFM_cAll_d29_mKR1_ch09.pkl -gWA_sFM_cAll_d25_mWA4_ch05.pkl -gJS_sFM_cAll_d03_mJS1_ch02.pkl -gLH_sFM_cAll_d18_mLH3_ch18.pkl -gJB_sFM_cAll_d07_mJB5_ch06.pkl -gMH_sFM_cAll_d22_mMH3_ch04.pkl -gPO_sFM_cAll_d10_mPO3_ch07.pkl -gBR_sFM_cAll_d04_mBR2_ch03.pkl -gLO_sFM_cAll_d13_mLO3_ch04.pkl -gKR_sFM_cAll_d29_mKR0_ch08.pkl -gPO_sFM_cAll_d12_mPO3_ch18.pkl From fce047faeed61dab7e7a9b38228d9936f9ebb8f9 Mon Sep 17 00:00:00 2001 From: Kelly-an <166104677+Kelly-an@users.noreply.github.com> Date: Tue, 11 Jun 2024 13:37:56 +0200 Subject: [PATCH 14/14] Delete notebook/chore_per_song.csv --- notebook/chore_per_song.csv | 62 ------------------------------------- 1 file changed, 62 deletions(-) delete mode 100644 notebook/chore_per_song.csv diff --git a/notebook/chore_per_song.csv b/notebook/chore_per_song.csv deleted file mode 100644 index f1f0c6a..0000000 --- a/notebook/chore_per_song.csv +++ /dev/null @@ -1,62 +0,0 @@ -4,name -mBR0,gBR_sFM_cAll_d04_mBR0_ch01.pkl -mBR1,gBR_sFM_cAll_d04_mBR1_ch02.pkl -mBR2,gBR_sFM_cAll_d06_mBR2_ch16.pkl -mBR3,gBR_sFM_cAll_d06_mBR3_ch17.pkl -mBR4,gBR_sFM_cAll_d05_mBR4_ch13.pkl -mBR5,gBR_sFM_cAll_d05_mBR5_ch12.pkl -mHO0,gHO_sFM_cAll_d20_mHO0_ch08.pkl -mHO1,gHO_sFM_cAll_d20_mHO1_ch09.pkl -mHO2,gHO_sFM_cAll_d19_mHO2_ch07.pkl -mHO3,gHO_sFM_cAll_d21_mHO3_ch21.pkl -mHO4,gHO_sFM_cAll_d21_mHO4_ch19.pkl -mHO5,gHO_sFM_cAll_d20_mHO5_ch13.pkl -mJB0,gJB_sFM_cAll_d07_mJB0_ch01.pkl -mJB1,gJB_sFM_cAll_d07_mJB1_ch02.pkl -mJB2,gJB_sFM_cAll_d08_mJB2_ch10.pkl -mJB3,gJB_sFM_cAll_d08_mJB3_ch11.pkl -mJB4,gJB_sFM_cAll_d09_mJB4_ch19.pkl -mJB5,gJB_sFM_cAll_d08_mJB5_ch14.pkl -mJS0,gJS_sFM_cAll_d01_mJS0_ch01.pkl -mJS1,gJS_sFM_cAll_d02_mJS1_ch02.pkl -mJS2,gJS_sFM_cAll_d03_mJS2_ch03.pkl -mJS3,gJS_sFM_cAll_d03_mJS3_ch04.pkl -mJS4,gJS_sFM_cAll_d02_mJS4_ch09.pkl -mJS5,gJS_sFM_cAll_d03_mJS5_ch13.pkl -mKK1,gJB_sFM_cAll_d07_mKK1_ch01.pkl -mKR0,gKR_sFM_cAll_d30_mKR0_ch15.pkl -mKR1,gKR_sFM_cAll_d30_mKR1_ch16.pkl -mKR2,gKR_sFM_cAll_d28_mKR2_ch03.pkl -mKR3,gKR_sFM_cAll_d30_mKR3_ch18.pkl -mKR4,gKR_sFM_cAll_d28_mKR4_ch05.pkl -mKR5,gKR_sFM_cAll_d28_mKR5_ch06.pkl -mLH0,gLH_sFM_cAll_d17_mLH0_ch14.pkl -mLH1,gLH_sFM_cAll_d18_mLH1_ch16.pkl -mLH2,gLH_sFM_cAll_d18_mLH2_ch17.pkl -mLH3,gLH_sFM_cAll_d16_mLH3_ch04.pkl -mLH4,gLH_sFM_cAll_d16_mLH4_ch05.pkl -mLH5,gLH_sFM_cAll_d16_mLH5_ch06.pkl -mLO0,gLO_sFM_cAll_d13_mLO0_ch01.pkl -mLO1,gLO_sFM_cAll_d13_mLO1_ch02.pkl -mLO2,gLO_sFM_cAll_d15_mLO2_ch17.pkl -mLO3,gLO_sFM_cAll_d14_mLO3_ch11.pkl -mLO4,gLO_sFM_cAll_d15_mLO4_ch19.pkl -mLO5,gLO_sFM_cAll_d13_mLO5_ch06.pkl -mMH0,gMH_sFM_cAll_d23_mMH0_ch08.pkl -mMH1,gMH_sFM_cAll_d23_mMH1_ch09.pkl -mMH2,gMH_sFM_cAll_d24_mMH2_ch17.pkl -mMH3,gMH_sFM_cAll_d23_mMH3_ch11.pkl -mMH4,gMH_sFM_cAll_d23_mMH4_ch12.pkl -mMH5,gMH_sFM_cAll_d23_mMH5_ch13.pkl -mPO0,gPO_sFM_cAll_d11_mPO0_ch08.pkl -mPO1,gPO_sFM_cAll_d11_mPO1_ch09.pkl -mPO2,gPO_sFM_cAll_d11_mPO2_ch10.pkl -mPO3,gPO_sFM_cAll_d11_mPO3_ch11.pkl -mPO4,gPO_sFM_cAll_d11_mPO4_ch12.pkl -mPO5,gPO_sFM_cAll_d11_mPO5_ch13.pkl -mWA0,gWA_sFM_cAll_d25_mWA0_ch01.pkl -mWA1,gWA_sFM_cAll_d25_mWA1_ch02.pkl -mWA2,gWA_sFM_cAll_d27_mWA2_ch17.pkl -mWA3,gWA_sFM_cAll_d27_mWA3_ch18.pkl -mWA4,gWA_sFM_cAll_d26_mWA4_ch12.pkl -mWA5,gWA_sFM_cAll_d27_mWA5_ch20.pkl