diff --git a/setup.py b/setup.py index c2b50c9..9e7bb7a 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,9 @@ + "tierpsytools.hydra.read_imgstore_extradata:" \ + "hydra_sensordata_report", "platechecker=" \ - + "tierpsytools.hydra.platechecker:platechecker" + + "tierpsytools.hydra.platechecker:platechecker", + "lawn_timelapse=" \ + + "tierpsytools.hydra.lawn_timelapse:lawn_timelapse" ] }, ) diff --git a/tierpsytools.egg-info/PKG-INFO b/tierpsytools.egg-info/PKG-INFO index 2438ede..5e6afff 100644 --- a/tierpsytools.egg-info/PKG-INFO +++ b/tierpsytools.egg-info/PKG-INFO @@ -1,6 +1,75 @@ Metadata-Version: 2.1 Name: tierpsytools Version: 0.1.dev0 +<<<<<<< HEAD +Summary: UNKNOWN +Home-page: UNKNOWN +Author: UNKNOWN +Author-email: UNKNOWN +License: UNKNOWN +Description: # tierpsytools + + tierpsytools is a Python library for dealing with metadata in worm screening experiments and tierpsy feature processing. + + ## Installation + It is recommended to create an environmemt to install tierpsytools in. In conda you can create an environmnet called tierpsytools with: + + ```bash + conda create -n tierpsytools + ``` + + Activate the environment with: + + ```bash + source activate tierpsytools + ``` + + To install tierpsytools clone the repository from github: + + ```bash + git clone https://github.com/Tierpsy/tierpsy-tools-python.git + ``` + + Go in the tierpsy-tools-python directory: + + ```bash + cd tierpsy-tools-python + ``` + + Firt, install the dependences with: + + ```bash + conda install --file requirements.txt + pip install imgstore + ``` + + Then install the tierpsytools package with: + + ```bash + pip install -e . + ``` + + + ## Usage + + You can import tierpsytools and use modules and functions as in: + + ```python + import tierpsytools + + filtered_features = tierpsytools.filter_features.drop_ventrally_signed(features) + ``` + + or import modules and functions from tierpsytools as in: + + ```python + from tierpsytools.filter_features import drop_ventrally_signed + + filtered_features = drop_ventrally_signed(features) + ``` + +Platform: UNKNOWN +======= # tierpsytools @@ -62,3 +131,4 @@ from tierpsytools.filter_features import drop_ventrally_signed filtered_features = drop_ventrally_signed(features) ``` +>>>>>>> upstream/master diff --git a/tierpsytools/hydra/lawn_timelapse.py b/tierpsytools/hydra/lawn_timelapse.py index 279c9ec..d6c6d18 100755 --- a/tierpsytools/hydra/lawn_timelapse.py +++ b/tierpsytools/hydra/lawn_timelapse.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- """ Timelapse Hydra 96-well (RawVideos) -- Make a timelapse video from the average frames of multiple short videos +- Make a timelapse video from the first frames of multiple short videos eg. for investiagting lawn growth rate over time @author: sm5911 @@ -19,14 +19,14 @@ from tqdm import tqdm from pathlib import Path from matplotlib import pyplot as plt -from tierpsy.analysis.split_fov.helper import serial2channel -from tierpsy.analysis.compress.selectVideoReader import selectVideoReader from tierpsytools.hydra import CAM2CH_df -from tierpsytools.hydra.hydra_filenames_helper import parse_camera_serial +from tierpsytools.hydra.hydra_filenames_helper import parse_camera_serial, serial2channel from tierpsytools.plot.plot_plate_trajectories_with_raw_video_background import CH2PLATE_dict +from tierpsytools.hydra.read_imgstore_extradata import ExtraDataReader +from tierpsytools.hydra.platechecker import get_frame_from_raw #%% Functions - + def get_video_list(RAWVIDEO_DIR, EXP_DATES=None, video_list_save_path=None): """ Search directory for 'metadata.yaml' video files and return as a list """ @@ -49,33 +49,47 @@ def get_video_list(RAWVIDEO_DIR, EXP_DATES=None, video_list_save_path=None): return video_list -def average_frame_yaml(metadata_yaml_path): - """ Return the average of the frames in a given 'metadata.yaml' video """ +def first_frame_yaml(metadata_yaml_path): - vid = selectVideoReader(str(metadata_yaml_path)) - frames = vid.read() - - avg_frame = np.mean(frames, axis=0) - - return avg_frame + # get info from raw video + edr = ExtraDataReader(str(metadata_yaml_path)) + + frame = get_frame_from_raw(edr) + + return frame + +# ============================================================================= +# def average_frame_yaml(metadata_yaml_path): +# """ Return the average of the frames in a given 'metadata.yaml' video """ +# +# from tierpsy.analysis.compress.selectVideoReader import selectVideoReader +# +# vid = selectVideoReader(str(metadata_yaml_path)) +# frames = vid.read() +# +# avg_frame = np.mean(frames, axis=0) +# +# return avg_frame +# ============================================================================= def save_avg_frames_for_timelapse(video_list, SAVE_DIR): - """ Take the average frame from each video and save to file """ + """ Take the first frame from each video and save to file """ video2frame_dict = {} - print('\nSaving average frame in %d videos...' % len(video_list)) + print('\nSaving first frame in %d videos...' % len(video_list)) for i, metadata_yaml_path in tqdm(enumerate(video_list)): metadata_yaml_path = Path(metadata_yaml_path) fstem = metadata_yaml_path.parent.name fname = fstem.replace('.','_') + '.tif' - savepath = Path(SAVE_DIR) / "average_frames" / fname - savepath.parent.mkdir(exist_ok=True) + savepath = Path(SAVE_DIR) / "first_frames" / fname + savepath.parent.mkdir(exist_ok=True, parents=True) - if not savepath.exists(): - avg_frame = average_frame_yaml(metadata_yaml_path) + if not savepath.exists(): + + avg_frame = first_frame_yaml(metadata_yaml_path) cv2.imwrite(str(savepath), avg_frame) video2frame_dict[str(metadata_yaml_path)] = str(savepath) @@ -190,6 +204,8 @@ def plate_frames_from_camera_frames(plate_frame_filename_dict, video2frame_dict, file_dict = get_rig_video_set(rig_video_set[0]) # gives channels as well assert sorted(file_dict.values()) == sorted([Path(i) for i in rig_video_set]) + # TODO: Use trajectory plotting function here + # define multi-panel figure columns = 3 rows = 2 @@ -221,7 +237,7 @@ def plate_frames_from_camera_frames(plate_frame_filename_dict, video2frame_dict, # get location of subplot for camera ax = axs[_loc] - # read average frame for rawvideopath + # read frame for rawvideopath av_frame_path = video2frame_dict[str(rawvideopath)] img = cv2.imread(av_frame_path) @@ -265,19 +281,17 @@ def make_video_from_frames(images_dir, video_name, plate_frame_filename_dict, fp outpath_video = Path(images_dir) / "{}.mp4".format(video_name) + print('Creating timelapse video...') video = cv2.VideoWriter(str(outpath_video), cv2.VideoWriter_fourcc(*'XVID'), fps, (width,height)) for imPath in tqdm(image_path_list): + # TODO: Create full plate view here when saving video.write(cv2.imread(str(imPath))) cv2.destroyAllWindows() video.release() - # video = moviepy.video.io.ImageSequenceClip.ImageSequenceClip(image_list, fps=fps) - # video.write_videofile(outpath_video) -#%% Main - -if __name__ == "__main__": +def lawn_timelapse(): parser = argparse.ArgumentParser(description='Create timelapse video from a series of RawVideos') parser.add_argument('--rawvideo_dir', help='Path to RawVideo directory', type=str, default=None) parser.add_argument('--video_list_path', help='Optional path to text file containing list of \ @@ -317,13 +331,13 @@ def make_video_from_frames(images_dir, video_name, plate_frame_filename_dict, fp print("%d videos found." % len(video_list)) - # save average frames for timelapse + # save camera frames for timelapse video2frame_dict = save_avg_frames_for_timelapse(video_list, args.save_dir) plate_frame_filename_dict = match_plate_frame_filenames(raw_video_path_list=video_list, join_across_days=args.join_days) - # create frames for timelapse (96-well) + # create plate frames for timelapse (96-well) plate_frames_from_camera_frames(plate_frame_filename_dict, video2frame_dict, args.save_dir) # create timelapse video @@ -333,3 +347,11 @@ def make_video_from_frames(images_dir, video_name, plate_frame_filename_dict, fp plate_frame_filename_dict=plate_frame_filename_dict, fps=args.fps) +# TODO: Could check for no errors, then clean up the unneeded files +# TODO: File timestamp in top right of video + +#%% Main + +if __name__ == "__main__": + lawn_timelapse() + \ No newline at end of file diff --git a/tierpsytools/plot/plot_plate_from_raw_video.py b/tierpsytools/plot/plot_plate_from_raw_video.py new file mode 100755 index 0000000..9be274b --- /dev/null +++ b/tierpsytools/plot/plot_plate_from_raw_video.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +A script to plot the full 96-well plate from a given RawVideo corresponding to a single camera view +Provide a RawVideo filepath and a plot will be produced of the entire 96-well plate +(imaged under 6 cameras simultaneously) for the first frame of the video + +@author: sm5911 +@date: 31/05/2023 + +""" + +#%% Imports + +import cv2 +import argparse +import numpy as np +from tqdm import tqdm +from pathlib import Path +from matplotlib import pyplot as plt +from tierpsytools.hydra.hydra_filenames_helper import CAM2CH_df, serial2channel, parse_camera_serial + +#%% Globals + +# Channel-to-plate mapping dictionary {'channel' : ((ax array location), rotate)} +CH2PLATE_dict = {'Ch1':((0,0),True), + 'Ch2':((1,0),False), + 'Ch3':((0,1),True), + 'Ch4':((1,1),False), + 'Ch5':((0,2),True), + 'Ch6':((1,2),False)} + +EXAMPLE_RAW_VIDEO_PATH = "/Volumes/hermes$/Saul/Keio_Screen/Data/Keio_Screen_Initial/RawVideos/20210406/keio_rep1_run1_bluelight_20210406_132006.22956809/000000.mp4" +FRAME = 0 +DPI = 900 + +#%% Functions + +def get_video_set(videofilepath): + """ Get the set of filenames of the featuresN results files that belong to + the same 96-well plate that was imaged under that rig """ + + # get camera serial from filename + camera_serial = parse_camera_serial(videofilepath) + + # get list of camera serials for that hydra rig + hydra_rig = CAM2CH_df.loc[CAM2CH_df['camera_serial'] == camera_serial, 'rig'] + rig_df = CAM2CH_df[CAM2CH_df['rig'] == hydra_rig.values[0]] + camera_serial_list = list(rig_df['camera_serial']) + + # extract filename stem + file_stem = str(videofilepath).split('.' + camera_serial)[0] + name = videofilepath.name + + # get paths to RawVideo files + file_dict = {} + for camera_serial in camera_serial_list: + channel = serial2channel(camera_serial) + _loc, rotate = CH2PLATE_dict[channel] + + video_path = Path(file_stem + '.' + camera_serial) / name + + file_dict[channel] = video_path + + return file_dict + +def plot_plate(video_dict, save_path, frame=0, dpi=900): # frame = 'all' + """ Tile first frame of raw videos for plate to create a single plot of the full 96-well plate, + correcting for camera orientation """ + + # define multi-panel figure + columns = 3 + rows = 2 + h_in = 4 + x_off_abs = (3600-3036) / 3036 * h_in + x = columns * h_in + x_off_abs + y = rows * h_in + + x_offset = x_off_abs / x # for bottom left image + width = (1-x_offset) / columns # for all but top left image + width_tl = width + x_offset # for top left image + height = 1/rows # for all images + + plt.close('all') + fig, axs = plt.subplots(rows, columns, figsize=[x,y]) + + errlog = [] + # print("Extracting frames...") + for channel, video_path in video_dict.items(): + + _loc, rotate = CH2PLATE_dict[channel] + _ri, _ci = _loc + + # create bbox for image layout in figure + if (_ri == 0) and (_ci == 0): + # first image, bbox slightly shifted + bbox = [0, height, width_tl, height] + else: + # other images + bbox = [x_offset + width * _ci, height * (rows - (_ri + 1)), width, height] + + # get location of subplot for camera + ax = axs[_loc] + + vidcap = cv2.VideoCapture(str(video_path)) + success, img = vidcap.read() + if success: + if rotate: + img = np.rot90(img, 2) + + ax.imshow(img, cmap='gray', vmin=0, vmax=255) + + else: + print("WARNING: Could not read video: '%s'" % (video_path)) + errlog.append(video_path) + + # set image position in figure + ax.set_position(bbox) + + ax.axes.get_xaxis().set_visible(False) + ax.axes.get_yaxis().set_visible(False) + + if len(errlog) > 0: + print(errlog) + + if save_path: + Path(save_path).parent.mkdir(exist_ok=True, parents=True) + fig.savefig(save_path, bbox_inches='tight', pad_inches=0, dpi=dpi) + + return + +def plot_plates_from_metadata(metadata, save_dir=None, dpi=DPI): + + # subset metadata for prestim videos only (to avoid duplicates for each plate) + prestim_videos = [i for i in metadata['imgstore_name'] if 'prestim' in i] + assert len(prestim_videos) > 0 + metadata = metadata[metadata['imgstore_name'].isin(prestim_videos)] + + # get list of video filenames for each plate + grouped = metadata.groupby('imaging_plate_id') + plate_list = metadata['imaging_plate_id'].unique() + + for i, plate in enumerate(tqdm(plate_list, initial=1)): + print('Plotting plate %d/%d: %s' % (i+1, len(plate_list), plate)) + meta = grouped.get_group(plate) + + video_dict = get_video_set(Path(meta['filename'].iloc[0]) / '000000.mp4') + if not all(str(i) in sorted(meta['filename']) for i in + [str(i.parent) for i in video_dict.values()]): + print("WARNING! %d camera videos missing from metadata:" % (len(video_dict) - + len(meta['filename']))) + # find symmetric difference (^) between both sets + # (all elements that appear only in set a or in set b, but not both) + missing_vids = set([str(i.parent) for i in video_dict.values()]) ^ set(meta['filename']) + for i in missing_vids: + print(i) + + # plot full plate view for each plate + save_dir = list(video_dict.values())[0].parent.parent if save_dir is None else save_dir + save_path = Path(save_dir) / 'plate_view' / 'plate_{}.png'.format(plate) + if not save_path.exists(): + plot_plate(video_dict=video_dict, save_path=save_path, frame=0, dpi=dpi) + else: + print('File already exists. Skipping file.') + + return + +#%% Main + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument('-f', '--raw_video_path', + help="Path to a RawVideo file (single camera video) of plate to plot", + type=str, default=EXAMPLE_RAW_VIDEO_PATH) + parser.add_argument('-s', '--save_dir', help="Path to save directory", type=str, default=None) + args = parser.parse_args() + + VIDEO_PATH = Path(args.raw_video_path) + SAVE_DIR = None if args.save_dir is None else Path(args.save_dir) + SAVE_PATH = (VIDEO_PATH.parent if SAVE_DIR is None else SAVE_DIR) / (VIDEO_PATH.stem + '.jpg') + + VIDEO_DICT = get_video_set(VIDEO_PATH) + + print("Plotting plate for %s" % str(VIDEO_PATH)) + plot_plate(video_dict=VIDEO_DICT, + save_path=SAVE_PATH, + frame=FRAME, + dpi=DPI) + + \ No newline at end of file diff --git a/tierpsytools/plot/plot_plate_trajectories_with_raw_video_background.py b/tierpsytools/plot/plot_plate_trajectories_with_raw_video_background.py index 0f2cda5..027b024 100755 --- a/tierpsytools/plot/plot_plate_trajectories_with_raw_video_background.py +++ b/tierpsytools/plot/plot_plate_trajectories_with_raw_video_background.py @@ -18,15 +18,12 @@ import cv2 import sys import h5py -import tqdm import argparse import pandas as pd from matplotlib import pyplot as plt from pathlib import Path - -from tierpsy.analysis.split_fov.FOVMultiWellsSplitter import FOVMultiWellsSplitter -from tierpsy.analysis.split_fov.helper import CAM2CH_df, serial2channel, parse_camera_serial -from tierpsy.analysis.compress.selectVideoReader import selectVideoReader +from tqdm import tqdm +from tierpsytools.hydra.hydra_filenames_helper import CAM2CH_df, serial2channel, parse_camera_serial #%% Channel-to-plate mapping dictionary (global) @@ -136,6 +133,9 @@ def feat2raw(featfilepath): def get_frame_from_raw(rawvidname): + + from tierpsy.analysis.compress.selectVideoReader import selectVideoReader + vid = selectVideoReader(str(rawvidname)) status, frame = vid.read_frame(0) assert status == 1, f'Something went wrong while reading {rawvidname}' @@ -177,6 +177,8 @@ def plot_plate_trajectories(featurefilepath, saveDir=None, downsample=10, fov= F """ Tile plots and merge into a single plot for the entire 96-well plate, correcting for camera orientation. """ + from tierpsy.analysis.split_fov.FOVMultiWellsSplitter import FOVMultiWellsSplitter + file_dict = get_video_set(featurefilepath) # define multi-panel figure @@ -251,8 +253,10 @@ def plot_plate_trajectories_from_filenames_summary(filenames_path, saveDir): """ Plot plate trajectories for all files in Tierpsy filenames summaries 'filenames_path', and save results to 'saveDir' """ + from tierpsytools.read_data.hydra_metadata import _get_filename_column + filenames_df = pd.read_csv(filenames_path, comment='#') - filenames_list = filenames_df[filenames_df['is_good']==True]['file_name'] + filenames_list = filenames_df[filenames_df['is_good']==True][_get_filename_column(filenames_path)] filestem_list = [] featurefile_list = [] @@ -282,17 +286,20 @@ def plot_plate_trajectories_from_filenames_summary(filenames_path, saveDir): # default to example file if none given parser.add_argument("--input", help="input file path (featuresN)", default=example_featuresN) - # default to input's grandparent if non given + # default to input's grandparent if none given known_args = parser.parse_known_args() parser.add_argument("--output", help="output directory path (for saving)", default=Path(known_args[0].input).parent.parent) - # parser.add_argument("--downsample", help="downsample trajectory data by plotting the worm centroid for every 'nth' frame", # default=10) args = parser.parse_args() print("Input file:", args.input) print("Output directory:", args.output) +<<<<<<< HEAD + plot_plate_trajectories(args.input, saveDir=args.output, downsample=10) +======= plot_plate_trajectories(args.input, saveDir=args.output, downsample=10, fov=False) +>>>>>>> upstream/master diff --git a/tierpsytools/preprocessing/filter_data.py b/tierpsytools/preprocessing/filter_data.py index 085e1ef..19e0189 100644 --- a/tierpsytools/preprocessing/filter_data.py +++ b/tierpsytools/preprocessing/filter_data.py @@ -198,7 +198,6 @@ def filter_nan_inf(feat, threshold, axis, verbose=True): import numpy as np sn = [(feat.shape[0], 'samples'), (feat.shape[1], 'features')] - nanRatio = np.sum(np.logical_or(np.isnan(feat), np.isinf(feat)), axis=axis) / np.size(feat, axis=axis) if axis==0: