Skip to content

Monika user detection #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
monikascholz wants to merge 26 commits into
base: master
Choose a base branch
from
Open

Monika user detection #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
dece99d
adding a gitignore to ignore binary stuff
aleifer Jul 10, 2019
70c2778
fixed paths in runAnalysis for my mac
aleifer Jul 10, 2019
ea6fc5c
updated paths for S4.py
aleifer Jul 10, 2019
48b39c5
Miminum set of changes that allow me to regenerate figures
aleifer Oct 31, 2019
66baa05
Changed GCAMP smoothing from 6 to 5.
aleifer Oct 31, 2019
ea64fc8
Turned off periodogram
aleifer Oct 31, 2019
73091b2
photobleaching correction by dividing off exponential fit
aleifer Nov 7, 2019
d26ead8
Adjusted the gaussian window used to calculate theta dot from 100 to 75
aleifer Nov 7, 2019
44aeca5
Improved robustness and fixed errors in photobleaching correction
aleifer Nov 7, 2019
5242d48
Place holder for outlier detection and NaNing
aleifer Nov 9, 2019
f5570a0
typo
aleifer Nov 9, 2019
28adc7c
nanOutliers function. no errors, but not fully tested.
aleifer Nov 14, 2019
d42483d
Now pipeline uses raw, does photobleaching, and outlier removal
aleifer Nov 14, 2019
467a92b
Made an if statement to turn off new photobleaching and outlier detec…
aleifer Nov 15, 2019
3830991
Changed folder directories of data quality check to make it run.
aleifer Nov 15, 2019
1846ecc
Adjusted the parameters back to the original
aleifer Nov 15, 2019
a3e24f9
Set meta run analysis to only deal with moving worms for now
aleifer Nov 15, 2019
3f0e900
Doing some tests to find out why fig2 only generates R2=0.44, no 0.49
aleifer Nov 15, 2019
8745a93
updated stamp on figure w/ provenance
aleifer Nov 15, 2019
bed516f
Now updated the provenance module to include branch name
aleifer Nov 21, 2019
41bc0de
Merge branch 'minimumLocal' into streamline
aleifer Nov 21, 2019
4c6b01c
testing hampel filter with debug output
aleifer Nov 21, 2019
d2d11d1
Monika's speedy hampel filter.
aleifer Nov 22, 2019
c49bf90
Fixed algebra error in photobleaching correction
aleifer Nov 22, 2019
353b98a
Debugging: python photobleaching and nan-ing aversely affects R^2
aleifer Nov 22, 2019
eea8b82
add automatic detection of user and modify all imports to conform
monikascholz Dec 13, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
*.pyc
build/
13 changes: 8 additions & 5 deletions figures/main/fig1_figManifold.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,12 @@
import numpy as np
import matplotlib as mpl
from sklearn.feature_selection import mutual_info_classif
import os
#
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import prediction.dataHandler as dh
from prediction import userTracker
# deliberate import all!
from prediction.stylesheet import *
# suddenly this isn't imported from stylesheet anymore...
Expand All @@ -29,10 +31,11 @@
data = {}
for typ in ['AML18', 'AML32']:
for condition in ['immobilized']:# ['moving', 'immobilized', 'chip']:
folder = '../../{}_{}/'.format(typ, condition)
dataLog = '../../{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition)
outLoc = "../../Analysis/{}_{}_results.hdf5".format(typ, condition)
outLocData = "../../Analysis/{}_{}.hdf5".format(typ, condition)
path = userTracker.dataPath()
folder = os.path.join(path, '{}_{}/'.format(typ, condition))
dataLog = os.path.join(path,'{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition))
outLoc = os.path.join(path, 'Analysis/{}_{}_results.hdf5'.format(typ, condition))
outLocData = os.path.join(path,'/Analysis/{}_{}.hdf5'.format(typ, condition))
try:
# load multiple datasets
dataSets = dh.loadDictFromHDF(outLocData)
Expand Down Expand Up @@ -140,4 +143,4 @@


#ax.view_init(elev=-40, azim=90)
plt.show()
plt.show()
10 changes: 6 additions & 4 deletions figures/main/fig1_figManifoldMoving.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import prediction.dataHandler as dh
from prediction import userTracker
# deliberate import all!
from prediction.stylesheet import *
# suddenly this isn't imported from stylesheet anymore...
Expand All @@ -29,10 +30,11 @@
data = {}
for typ in ['AML18', 'AML32']:
for condition in ['moving']:# ['moving', 'immobilized', 'chip']:
folder = '../../{}_{}/'.format(typ, condition)
dataLog = '../../{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition)
outLoc = "../../Analysis/{}_{}_results.hdf5".format(typ, condition)
outLocData = "../../Analysis/{}_{}.hdf5".format(typ, condition)
path = userTracker.dataPath()
folder = os.path.join(path, '{}_{}/'.format(typ, condition))
dataLog = os.path.join(path,'{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition))
outLoc = os.path.join(path, 'Analysis/{}_{}_results.hdf5'.format(typ, condition))
outLocData = os.path.join(path,'/Analysis/{}_{}.hdf5'.format(typ, condition))
try:
# load multiple datasets
dataSets = dh.loadDictFromHDF(outLocData)
Expand Down
10 changes: 6 additions & 4 deletions figures/main/fig1v3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#import singlePanels as sp
#import makePlots as mp
import prediction.dataHandler as dh
from prediction import userTracker

# suddenly this isn't imported from stylesheet anymore...
mpl.rcParams["axes.labelsize"] = 14
Expand All @@ -42,10 +43,11 @@
data = {}
for typ in ['AML32', 'AML18', 'AML70', 'AML175', 'Special']:
for condition in ['chip', 'moving', 'immobilized', 'transition']:# ['moving', 'immobilized', 'chip']:
folder = '../../{}_{}/'.format(typ, condition)
dataLog = '../../{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition)
outLoc = "../../Analysis/{}_{}_results.hdf5".format(typ, condition)
outLocData = "../../Analysis/{}_{}.hdf5".format(typ, condition)
path = userTracker.dataPath()
folder = os.path.join(path, '{}_{}/'.format(typ, condition))
dataLog = os.path.join(path,'{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition))
outLoc = os.path.join(path, 'Analysis/{}_{}_results.hdf5'.format(typ, condition))
outLocData = os.path.join(path,'/Analysis/{}_{}.hdf5'.format(typ, condition))

try:
# load multiple datasets
Expand Down
11 changes: 7 additions & 4 deletions figures/main/fig2_expVarslm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@
"""
import numpy as np
import matplotlib as mpl
import os
#
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from scipy.ndimage.filters import gaussian_filter1d
#
#import makePlots as mp
from prediction import dataHandler as dh
from prediction import userTracker
# deliberate import all!
from prediction.stylesheet import *
from scipy.stats import pearsonr
Expand All @@ -34,10 +36,11 @@
data = {}
for typ in ['AML32', 'AML18', 'AML175', 'AML70']:
for condition in ['moving', 'chip']:# ['moving', 'immobilized', 'chip']:
folder = '../../{}_{}/'.format(typ, condition)
dataLog = '../../{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition)
outLoc = "../../Analysis/{}_{}_results.hdf5".format(typ, condition)
outLocData = "../../Analysis/{}_{}.hdf5".format(typ, condition)
path = userTracker.dataPath()
folder = os.path.join(path, '{}_{}/'.format(typ, condition))
dataLog = os.path.join(path,'{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition))
outLoc = os.path.join(path, 'Analysis/{}_{}_results.hdf5'.format(typ, condition))
outLocData = os.path.join(path,'/Analysis/{}_{}.hdf5'.format(typ, condition))

try:
# load multiple datasets
Expand Down
17 changes: 13 additions & 4 deletions figures/main/fig2v3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@
"""
import numpy as np
import matplotlib as mpl
import os
#
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from scipy.ndimage.filters import gaussian_filter1d
#
#import makePlots as mp
import prediction.dataHandler as dh
from prediction import userTracker
# deliberate import all!
from prediction.stylesheet import *
from scipy.stats import pearsonr
Expand All @@ -33,10 +35,11 @@
data = {}
for typ in ['AML32', 'AML18', 'AML175', 'AML70']:
for condition in ['moving', 'chip']:# ['moving', 'immobilized', 'chip']:
folder = '../../{}_{}/'.format(typ, condition)
dataLog = '../../{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition)
outLoc = "../../Analysis/{}_{}_results.hdf5".format(typ, condition)
outLocData = "../../Analysis/{}_{}.hdf5".format(typ, condition)
path = userTracker.dataPath()
folder = os.path.join(path, '{}_{}/'.format(typ, condition))
dataLog = os.path.join(path,'{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition))
outLoc = os.path.join(path, 'Analysis/{}_{}_results.hdf5'.format(typ, condition))
outLocData = os.path.join(path,'/Analysis/{}_{}.hdf5'.format(typ, condition))

try:
# load multiple datasets
Expand Down Expand Up @@ -218,6 +221,7 @@
ax7.text(-140, 0, 'Velocity\n(rad/s)', color = R1, rotation=0, verticalalignment='center', fontsize=12, multialignment='center')
ax8.text(-160, 0, 'Body \n curvature \n(a.u.)', color = B1, rotation=0, verticalalignment='center', fontsize=12, multialignment='center')


# move axis to the right
ax7.yaxis.tick_right()
ax8.yaxis.tick_right()
Expand Down Expand Up @@ -493,6 +497,11 @@
plt.show()
# get all the weights for the different samples


import prediction.provenance as prov
prov.stamp(axV,.1,.3)


#for typ, colors, ax in zip(['AML32', 'AML18'], [colorsExp, colorCtrl], [ax11, ax12]):
# for condition in ['moving', 'immobilized']:
# key = '{}_{}'.format(typ, condition)
Expand Down
11 changes: 6 additions & 5 deletions figures/main/fig3v3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
#from matplotlib_venn import venn2
from scipy.cluster.hierarchy import dendrogram, linkage, fcluster, set_link_color_palette
import prediction.dataHandler as dh

from prediction import userTracker
from prediction.stylesheet import *
from prediction.pycpd import deformable_registration, rigid_registration

Expand Down Expand Up @@ -54,10 +54,11 @@ def registerWorms(R, X, dim=3, nr = True):
data = {}
for typ in ['AML32', 'AML18', 'AML70', 'AML175']:
for condition in ['moving', 'chip']:# ['moving', 'immobilized', 'chip']:
folder = '../../{}_{}/'.format(typ, condition)
dataLog = '../../{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition)
outLoc = "../../Analysis/{}_{}_results.hdf5".format(typ, condition)
outLocData = "../../Analysis/{}_{}.hdf5".format(typ, condition)
path = userTracker.dataPath()
folder = os.path.join(path, '{}_{}/'.format(typ, condition))
dataLog = os.path.join(path,'{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition))
outLoc = os.path.join(path, 'Analysis/{}_{}_results.hdf5'.format(typ, condition))
outLocData = os.path.join(path,'/Analysis/{}_{}.hdf5'.format(typ, condition))

try:
# load multiple datasets
Expand Down
11 changes: 6 additions & 5 deletions figures/main/fig4v3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
#

import prediction.dataHandler as dh

from prediction import userTracker
from prediction.stylesheet import *
# stats
from sklearn.metrics import r2_score,mean_squared_error
Expand Down Expand Up @@ -93,10 +93,11 @@ def main():
data = {}
for typ in ['AML32', 'AML70']:
for condition in ['moving', 'chip']:# ['moving', 'immobilized', 'chip']:
folder = '../../{}_{}/'.format(typ, condition)
dataLog = '../../{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition)
outLoc = "../../Analysis/{}_{}_results.hdf5".format(typ, condition)
outLocData = "../../Analysis/{}_{}.hdf5".format(typ, condition)
path = userTracker.dataPath()
folder = os.path.join(path, '{}_{}/'.format(typ, condition))
dataLog = os.path.join(path,'{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition))
outLoc = os.path.join(path, 'Analysis/{}_{}_results.hdf5'.format(typ, condition))
outLocData = os.path.join(path,'/Analysis/{}_{}.hdf5'.format(typ, condition))

try:
# load multiple datasets
Expand Down
12 changes: 7 additions & 5 deletions figures/supp/S1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from scipy.ndimage.filters import gaussian_filter1d
import matplotlib.ticker as mtick
import prediction.dataHandler as dh
from prediction import userTracker
# deliberate import all!
from prediction.stylesheet import *
# suddenly this isn't imported from stylesheet anymore...
Expand All @@ -34,11 +35,12 @@
data = {}
for typ in ['AML18', 'AML175', 'Special']:
for condition in ['chip', 'moving', 'immobilized', 'transition']:# ['moving', 'immobilized', 'chip']:
folder = '../../{}_{}/'.format(typ, condition)
dataLog = '../../{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition)
outLoc = "../../Analysis/{}_{}_results.hdf5".format(typ, condition)
outLocData = "../../Analysis/{}_{}.hdf5".format(typ, condition)

path = userTracker.dataPath()
folder = os.path.join(path, '{}_{}/'.format(typ, condition))
dataLog = os.path.join(path,'{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition))
outLoc = os.path.join(path, 'Analysis/{}_{}_results.hdf5'.format(typ, condition))
outLocData = os.path.join(path,'/Analysis/{}_{}.hdf5'.format(typ, condition))

try:
# load multiple datasets
dataSets = dh.loadDictFromHDF(outLocData)
Expand Down
13 changes: 7 additions & 6 deletions figures/supp/S10.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
#import svgutils as svg
#
import prediction.dataHandler as dh

from prediction import userTracker
from prediction.stylesheet import *
from prediction.pycpd import deformable_registration, rigid_registration

Expand Down Expand Up @@ -59,11 +59,12 @@ def registerWorms(Xref, X, dim=3):
data = {}
for typ in ['AML32', 'AML18', 'AML70', 'AML175']:
for condition in ['moving', 'chip']:# ['moving', 'immobilized', 'chip']:
folder = '../../{}_{}/'.format(typ, condition)
dataLog = '../../{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition)
outLoc = "../../Analysis/{}_{}_results.hdf5".format(typ, condition)
outLocData = "../../Analysis/{}_{}.hdf5".format(typ, condition)

path = userTracker.dataPath()
folder = os.path.join(path, '{}_{}/'.format(typ, condition))
dataLog = os.path.join(path,'{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition))
outLoc = os.path.join(path, 'Analysis/{}_{}_results.hdf5'.format(typ, condition))
outLocData = os.path.join(path,'/Analysis/{}_{}.hdf5'.format(typ, condition))

try:
# load multiple datasets
dataSets = dh.loadDictFromHDF(outLocData)
Expand Down
10 changes: 6 additions & 4 deletions figures/supp/S2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from scipy.ndimage.filters import gaussian_filter1d
import matplotlib.ticker as mtick
import prediction.dataHandler as dh
from prediction import userTracker
# deliberate import all!
from prediction.stylesheet import *
# suddenly this isn't imported from stylesheet anymore...
Expand All @@ -33,10 +34,11 @@
data = {}
for typ in ['AML32', 'AML18', 'AML70', 'AML175', 'Special']:
for condition in ['chip', 'moving', 'immobilized', 'transition']:# ['moving', 'immobilized', 'chip']:
folder = '../../{}_{}/'.format(typ, condition)
dataLog = '../../{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition)
outLoc = "../../Analysis/{}_{}_results.hdf5".format(typ, condition)
outLocData = "../../Analysis/{}_{}.hdf5".format(typ, condition)
path = userTracker.dataPath()
folder = os.path.join(path, '{}_{}/'.format(typ, condition))
dataLog = os.path.join(path,'{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition))
outLoc = os.path.join(path, 'Analysis/{}_{}_results.hdf5'.format(typ, condition))
outLocData = os.path.join(path,'/Analysis/{}_{}.hdf5'.format(typ, condition))

try:
# load multiple datasets
Expand Down
10 changes: 6 additions & 4 deletions figures/supp/S3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from scipy.ndimage.filters import gaussian_filter1d
import matplotlib.ticker as mtick
import prediction.dataHandler as dh
from prediction import userTracker
# deliberate import all!
from prediction.stylesheet import *

Expand All @@ -35,10 +36,11 @@
data = {}
for typ in ['AML32', 'Special']:
for condition in ['immobilized', 'transition']:# ['moving', 'immobilized', 'chip']:
folder = '../../{}_{}/'.format(typ, condition)
dataLog = '../../{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition)
outLoc = "../../Analysis/{}_{}_results.hdf5".format(typ, condition)
outLocData = "../../Analysis/{}_{}.hdf5".format(typ, condition)
path = userTracker.dataPath()
folder = os.path.join(path, '{}_{}/'.format(typ, condition))
dataLog = os.path.join(path,'{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition))
outLoc = os.path.join(path, 'Analysis/{}_{}_results.hdf5'.format(typ, condition))
outLocData = os.path.join(path,'/Analysis/{}_{}.hdf5'.format(typ, condition))

try:
# load multiple datasets
Expand Down
12 changes: 7 additions & 5 deletions figures/supp/S4.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import matplotlib.ticker as mtick
from scipy.stats import ttest_ind
from prediction import dataHandler as dh
from prediction import userTracker
# deliberate import all!
from prediction.stylesheet import *

Expand Down Expand Up @@ -64,11 +65,12 @@

data = {}
for typ in ['AML32', 'AML18', 'AML70', 'AML175']:
for condition in ['chip', 'moving', 'immobilized']:# ['moving', 'immobilized', 'chip']:
folder = '../../{}_{}/'.format(typ, condition)
dataLog = '../../{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition)
outLoc = "../../Analysis/{}_{}_results.hdf5".format(typ, condition)
outLocData = "../../Analysis/{}_{}.hdf5".format(typ, condition)
for condition in [ 'chip', 'moving', 'immobilized']:# ['moving', 'immobilized', 'chip']:
path = userTracker.dataPath()
folder = os.path.join(path, '{}_{}/'.format(typ, condition))
dataLog = os.path.join(path,'{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition))
outLoc = os.path.join(path, 'Analysis/{}_{}_results.hdf5'.format(typ, condition))
outLocData = os.path.join(path,'/Analysis/{}_{}.hdf5'.format(typ, condition))

try:
# load multiple datasets
Expand Down
11 changes: 7 additions & 4 deletions figures/supp/S5.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,12 @@
import numpy as np
import matplotlib as mpl
from sklearn.feature_selection import mutual_info_classif
import os
#
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import prediction.dataHandler as dh
from prediction import userTracker
# deliberate import all!
from prediction.stylesheet import *
# suddenly this isn't imported from stylesheet anymore...
Expand All @@ -29,10 +31,11 @@
data = {}
for typ in ['AML32', 'AML18', 'AML70', 'AML175']:
for condition in ['chip', 'moving', 'immobilized']:# ['moving', 'immobilized', 'chip']:
folder = '../../{}_{}/'.format(typ, condition)
dataLog = '../../{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition)
outLoc = "../../Analysis/{}_{}_results.hdf5".format(typ, condition)
outLocData = "../../Analysis/{}_{}.hdf5".format(typ, condition)
path = userTracker.dataPath()
folder = os.path.join(path, '{}_{}/'.format(typ, condition))
dataLog = os.path.join(path,'{0}_{1}/{0}_{1}_datasets.txt'.format(typ, condition))
outLoc = os.path.join(path, 'Analysis/{}_{}_results.hdf5'.format(typ, condition))
outLocData = os.path.join(path,'/Analysis/{}_{}.hdf5'.format(typ, condition))

try:
# load multiple datasets
Expand Down
Loading