From 1efef87962f93e95599617a27dcff430eedb2190 Mon Sep 17 00:00:00 2001 From: Marcus Breese Date: Sat, 21 Jul 2012 05:57:14 -0400 Subject: [PATCH 01/10] added support for virtualenv based installs Altered Makefile to call install.sh for default action Added install.sh to setup virtualenv, run python install, install deps added "miso" driver script to setup virtualenv and call the required miso.py script --- .gitignore | 1 + Makefile | 1 + install.sh | 21 +++++++++++++++++++++ miso | 44 ++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 5 +++++ 5 files changed, 72 insertions(+) create mode 100755 install.sh create mode 100755 miso create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 299f26c..1883ab0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.pyc splicing_0.1.tar.gz pysplicing-0.1.tar.gz +env/* diff --git a/Makefile b/Makefile index 70c61a6..e9b2ec8 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ all: Pythonpackage + ./install.sh ######################################################## diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..ceb9778 --- /dev/null +++ b/install.sh @@ -0,0 +1,21 @@ +#!/bin/bash +VIRTUALENV=`which virtualenv-2.7` +if [ "$VIRTUALENV" == "" ]; then + VIRTUALENV=`which virtualenv-2.6` +fi +if [ "$VIRTUALENV" == "" ]; then + VIRTUALENV=`which virtualenv` +fi +if [ "$VIRTUALENV" == "" ]; then + echo "Missing virtualenv!" + exit 1 +fi + +if [ ! -e env ]; then + echo "Initializing virtualenv folder (env)" + $VIRTUALENV env +fi +. env/bin/activate +python setup.py install --prefix env +pip install numpy +pip install -r requirements.txt diff --git a/miso b/miso new file mode 100755 index 0000000..8d5f901 --- /dev/null +++ b/miso @@ -0,0 +1,44 @@ +#!/bin/bash +function usage() { + echo "Usage: $(basename $0) command args..." + echo "" + echo "Valid commands: exon_utils" + echo " filter_events" + echo " index_gff" + echo " module_availability" + echo " pe_utils" + echo " plot" + echo " run_events_analysis" + echo " run" + echo " sam_to_bam" + exit 1 +} +DIR=$(dirname $0) +. $DIR/env/bin/activate + + +if [ ! "$1" ]; then + usage +fi + +cmd="$1" +shift + +if [ "$cmd" == "run" ]; then + python $DIR/env/bin/run_miso.py "$@" +elif [ -e $DIR/env/bin/$cmd.py ]; then + python $DIR/env/bin/$cmd.py "$@" +else + usage +fi + + +#exon_utils.py +#filter_events.py +#index_gff.py +#module_availability.py +#pe_utils.py +#plot.py +#run_events_analysis.py +#run_miso.py +#sam_to_bam.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a22a9ec --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +numpy +scipy +simplejson +pysam +matplotlib From ec6362e7b950895f7e27dc35a9bfd2147d52254b Mon Sep 17 00:00:00 2001 From: Marcus Breese Date: Sat, 21 Jul 2012 06:07:05 -0400 Subject: [PATCH 02/10] get actual DIR, not dirname $0... need to call python to get correct full path --- miso | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/miso b/miso index 8d5f901..77ce561 100755 --- a/miso +++ b/miso @@ -13,7 +13,8 @@ function usage() { echo " sam_to_bam" exit 1 } -DIR=$(dirname $0) +REAL=`python -c 'import os,sys;print os.path.realpath(sys.argv[1])' "$0"` +DIR=`dirname "$REAL"` . $DIR/env/bin/activate From e3013e59b232b9c463ffcebb5812eacd248f645b Mon Sep 17 00:00:00 2001 From: Marcus Breese Date: Fri, 27 Jul 2012 02:09:00 -0700 Subject: [PATCH 03/10] added ability to also run "run_miso.py" for example instead of just "run_miso" (both will work) --- miso | 2 ++ 1 file changed, 2 insertions(+) diff --git a/miso b/miso index 77ce561..2e3da76 100755 --- a/miso +++ b/miso @@ -29,6 +29,8 @@ if [ "$cmd" == "run" ]; then python $DIR/env/bin/run_miso.py "$@" elif [ -e $DIR/env/bin/$cmd.py ]; then python $DIR/env/bin/$cmd.py "$@" +elif [ -e $DIR/env/bin/$cmd ]; then + python $DIR/env/bin/$cmd "$@" else usage fi From 24afe34c377f8686f3ab5c3da7537adda6acf440 Mon Sep 17 00:00:00 2001 From: Marcus Breese Date: Fri, 27 Jul 2012 02:33:30 -0700 Subject: [PATCH 04/10] Replaced instances of "python" with "miso" when calling os.system --- .gitignore | 1 + misopy/run_events_analysis.py | 4 ++-- misopy/run_miso.py | 4 ++-- misopy/test_cluster.py | 6 ++++-- misopy/test_miso.py | 4 ++-- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 1883ab0..fc8c570 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ splicing_0.1.tar.gz pysplicing-0.1.tar.gz env/* +build/* \ No newline at end of file diff --git a/misopy/run_events_analysis.py b/misopy/run_events_analysis.py index 3f30310..a7f6047 100644 --- a/misopy/run_events_analysis.py +++ b/misopy/run_events_analysis.py @@ -47,7 +47,7 @@ def compute_all_genes_psi(gff_dir, bam_filename, read_len, output_dir, all_miso_cmds = [] for gene_id, gff_index_filename in gene_ids_to_gff_index.iteritems(): - miso_cmd = "python %s --compute-gene-psi \"%s\" \"%s\" %s %s --read-len %d " \ + miso_cmd = "miso %s --compute-gene-psi \"%s\" \"%s\" %s %s --read-len %d " \ %(miso_run, gene_id, gff_index_filename, bam_filename, output_dir, read_len) if paired_end != None: @@ -183,7 +183,7 @@ def compute_psi(sample_filenames, output_dir, event_type, read_len, overhang_len events_filename = events.output_file(results_output_dir, sample_label) # Run MISO on them - miso_cmd = 'python %s --compute-two-iso-psi %s %s --event-type %s --read-len %d --overhang-len %d ' \ + miso_cmd = 'miso %s --compute-two-iso-psi %s %s --event-type %s --read-len %d --overhang-len %d ' \ %(os.path.join(miso_path, 'run_miso.py'), events_filename, results_output_dir, diff --git a/misopy/run_miso.py b/misopy/run_miso.py index 9401891..38cf232 100644 --- a/misopy/run_miso.py +++ b/misopy/run_miso.py @@ -148,7 +148,7 @@ def run_two_iso_on_cluster(miso_path, events_filename, event_type, psi_outdir, num_jobs_per_batch = len(event_batch) print "Processing a batch of size %d events" %(num_jobs_per_batch) for event_name in event_batch: - miso_event_cmd = 'python %s --run-two-iso-event \"%s\" %s %s --event-type %s --read-len %d --overhang-len %d' \ + miso_event_cmd = 'miso %s --run-two-iso-event \"%s\" %s %s --event-type %s --read-len %d --overhang-len %d' \ %(os.path.join(miso_path, 'run_miso.py'), event_name, events_filename, @@ -181,7 +181,7 @@ def get_curr_script_cmd(): Get the invocation of the current script (with its command line arguments) as a full command for use in a script. """ - return 'python ' + get_current_args() + return 'miso %s' % get_current_args() def strip_option(cmd, option): """ diff --git a/misopy/test_cluster.py b/misopy/test_cluster.py index 1df31ae..40cde11 100644 --- a/misopy/test_cluster.py +++ b/misopy/test_cluster.py @@ -4,6 +4,7 @@ ## import unittest import os +import Settings class TestCluster(unittest.TestCase): """ @@ -13,7 +14,7 @@ def setUp(self): # Find out the current directory self.miso_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) self.tests_data_dir = os.path.join(self.miso_path, "test-data") - self.events_analysis_cmd = "python %s " %(os.path.join(self.miso_path, + self.events_analysis_cmd = "miso %s " %(os.path.join(self.miso_path, "run_events_analysis.py")) self.tests_output_dir = os.path.join(self.miso_path, "test-output") self.test_sam_filename = os.path.join(self.tests_data_dir, @@ -69,7 +70,7 @@ def test_cluster_gene_psi(self): # First index the GFF of interest gff_filename = os.path.join(self.gff_events_dir, "mm9", "genes", "Atp2b1.mm9.gff") gff_index_dir = os.path.join(self.gff_events_dir, "mm9", "indexed") - index_cmd = "python %s --index %s %s" %(self.index_gff_script, + index_cmd = "miso %s --index %s %s" %(self.index_gff_script, gff_filename, gff_index_dir) @@ -92,4 +93,5 @@ def test_cluster_gene_psi(self): if __name__ == '__main__': + Settings.load() unittest.main() diff --git a/misopy/test_miso.py b/misopy/test_miso.py index 77a8d2c..243448a 100644 --- a/misopy/test_miso.py +++ b/misopy/test_miso.py @@ -10,7 +10,7 @@ def setUp(self): # Find out the current directory self.miso_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) self.tests_data_dir = os.path.join(self.miso_path, "test-data") - self.events_analysis_cmd = "python %s " %(os.path.join(self.miso_path, + self.events_analysis_cmd = "miso %s " %(os.path.join(self.miso_path, "run_events_analysis.py")) self.tests_output_dir = os.path.join(self.miso_path, "test-output") self.test_sam_filename = os.path.join(self.tests_data_dir, @@ -29,7 +29,7 @@ def test_a_sam_to_bam(self): print "Testing conversion of SAM to BAM..." output_dir = os.path.join(self.tests_output_dir, "sam-output") - sam_to_bam_cmd = "python %s --convert %s %s" %(self.sam_to_bam_script, + sam_to_bam_cmd = "miso %s --convert %s %s" %(self.sam_to_bam_script, self.test_sam_filename, output_dir) print "Executing: %s" %(sam_to_bam_cmd) From 5e57584fed979f6591caffb72a24e9eecbdb7706 Mon Sep 17 00:00:00 2001 From: Marcus Breese Date: Fri, 27 Jul 2012 02:42:10 -0700 Subject: [PATCH 05/10] added some help for Macs and matplotlib --- install.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/install.sh b/install.sh index ceb9778..1211ecf 100755 --- a/install.sh +++ b/install.sh @@ -19,3 +19,7 @@ fi python setup.py install --prefix env pip install numpy pip install -r requirements.txt + +# +# If you're on a Mac, and matplotlib won't install, try this: +# LDFLAGS="-L/usr/X11/lib" CFLAGS="-I/usr/X11/include -I/usr/X11/include/freetype2 -I/usr/X11/include/libpng15" pip install matplotlib From f0e3b367bef51f125334e6a2ef57e09c5b50c9aa Mon Sep 17 00:00:00 2001 From: Marcus Breese Date: Fri, 27 Jul 2012 07:06:36 -0400 Subject: [PATCH 06/10] more options for commands to be found - to avoid larger changes in paths, etc... --- miso | 3 +++ 1 file changed, 3 insertions(+) diff --git a/miso b/miso index 2e3da76..5c40ded 100755 --- a/miso +++ b/miso @@ -31,7 +31,10 @@ elif [ -e $DIR/env/bin/$cmd.py ]; then python $DIR/env/bin/$cmd.py "$@" elif [ -e $DIR/env/bin/$cmd ]; then python $DIR/env/bin/$cmd "$@" +elif [ -e $cmd ]; then + python $cmd "$@" else + echo "Missing: $cmd" usage fi From 47997ab63d085ff8baa180455378a80ebcbc28a3 Mon Sep 17 00:00:00 2001 From: Marcus Breese Date: Fri, 27 Jul 2012 07:07:00 -0400 Subject: [PATCH 07/10] don't include test output in repo --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fc8c570..e2963be 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ splicing_0.1.tar.gz pysplicing-0.1.tar.gz env/* -build/* \ No newline at end of file +build/* +misopy/test-output/* From 315376c7f3efc9035f78d01f9be7e8d1a0747c00 Mon Sep 17 00:00:00 2001 From: Marcus Breese Date: Mon, 30 Jul 2012 00:55:52 -0700 Subject: [PATCH 08/10] Make virtualenv install optional - added 'make virtualenv' command --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index e9b2ec8..b7ce07d 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ all: Pythonpackage + +virtualenv: Pythonpackage ./install.sh ######################################################## From 19df5030034d01fcfe09ad05b6d7f24716381d80 Mon Sep 17 00:00:00 2001 From: Marcus Breese Date: Mon, 30 Jul 2012 01:20:18 -0700 Subject: [PATCH 09/10] Virtualenv support cleanup Added support for running MISO with and without virtualenv - the "miso" wrapper script adds an ENV variable if it is running... this way MISO can determine whether or not it should call 'python' or 'miso' when calling os.system(). To do this I added a get_miso_exec() method to settings.Settings. Added support for calling test scripts with miso driver script: miso $MISOHOME/misopy/test_miso.py --- miso | 30 ++++++++++++------------------ misopy/run_events_analysis.py | 9 +++++---- misopy/run_miso.py | 7 ++++--- misopy/settings.py | 6 ++++++ misopy/test_cluster.py | 8 ++++---- misopy/test_miso.py | 20 ++++++++++++-------- 6 files changed, 43 insertions(+), 37 deletions(-) diff --git a/miso b/miso index 5c40ded..d83eb63 100755 --- a/miso +++ b/miso @@ -11,6 +11,7 @@ function usage() { echo " run_events_analysis" echo " run" echo " sam_to_bam" + echo "" exit 1 } REAL=`python -c 'import os,sys;print os.path.realpath(sys.argv[1])' "$0"` @@ -25,26 +26,19 @@ fi cmd="$1" shift +export MISO_SHELL_EXEC="$0" + if [ "$cmd" == "run" ]; then python $DIR/env/bin/run_miso.py "$@" -elif [ -e $DIR/env/bin/$cmd.py ]; then - python $DIR/env/bin/$cmd.py "$@" -elif [ -e $DIR/env/bin/$cmd ]; then - python $DIR/env/bin/$cmd "$@" -elif [ -e $cmd ]; then - python $cmd "$@" +elif [ -e "$DIR/env/bin/$cmd.py" ]; then + python "$DIR/env/bin/$cmd.py" "$@" +elif [ -e "$DIR/env/bin/$cmd" ]; then + python "$DIR/env/bin/$cmd" "$@" +elif [ -e "$cmd" ]; then + # this is for hard-coded commands, such as used in the cluster code + # or to run test-scripts + python "$cmd" "$@" else - echo "Missing: $cmd" + echo "Unknown command: $cmd" usage fi - - -#exon_utils.py -#filter_events.py -#index_gff.py -#module_availability.py -#pe_utils.py -#plot.py -#run_events_analysis.py -#run_miso.py -#sam_to_bam.py diff --git a/misopy/run_events_analysis.py b/misopy/run_events_analysis.py index a7f6047..38caab4 100644 --- a/misopy/run_events_analysis.py +++ b/misopy/run_events_analysis.py @@ -47,8 +47,8 @@ def compute_all_genes_psi(gff_dir, bam_filename, read_len, output_dir, all_miso_cmds = [] for gene_id, gff_index_filename in gene_ids_to_gff_index.iteritems(): - miso_cmd = "miso %s --compute-gene-psi \"%s\" \"%s\" %s %s --read-len %d " \ - %(miso_run, gene_id, gff_index_filename, bam_filename, output_dir, + miso_cmd = "%s %s --compute-gene-psi \"%s\" \"%s\" %s %s --read-len %d " \ + %(Settings.get_miso_exec(), miso_run, gene_id, gff_index_filename, bam_filename, output_dir, read_len) if paired_end != None: # Run in paired-end mode @@ -183,8 +183,9 @@ def compute_psi(sample_filenames, output_dir, event_type, read_len, overhang_len events_filename = events.output_file(results_output_dir, sample_label) # Run MISO on them - miso_cmd = 'miso %s --compute-two-iso-psi %s %s --event-type %s --read-len %d --overhang-len %d ' \ - %(os.path.join(miso_path, 'run_miso.py'), + miso_cmd = '%s %s --compute-two-iso-psi %s %s --event-type %s --read-len %d --overhang-len %d ' \ + %(Settings.get_miso_exec(), + os.path.join(miso_path, 'run_miso.py'), events_filename, results_output_dir, event_type, diff --git a/misopy/run_miso.py b/misopy/run_miso.py index 38cf232..c8c8030 100644 --- a/misopy/run_miso.py +++ b/misopy/run_miso.py @@ -148,8 +148,9 @@ def run_two_iso_on_cluster(miso_path, events_filename, event_type, psi_outdir, num_jobs_per_batch = len(event_batch) print "Processing a batch of size %d events" %(num_jobs_per_batch) for event_name in event_batch: - miso_event_cmd = 'miso %s --run-two-iso-event \"%s\" %s %s --event-type %s --read-len %d --overhang-len %d' \ - %(os.path.join(miso_path, 'run_miso.py'), + miso_event_cmd = '%s %s --run-two-iso-event \"%s\" %s %s --event-type %s --read-len %d --overhang-len %d' \ + %(Settings.get_miso_exec(), + os.path.join(miso_path, 'run_miso.py'), event_name, events_filename, psi_outdir, @@ -181,7 +182,7 @@ def get_curr_script_cmd(): Get the invocation of the current script (with its command line arguments) as a full command for use in a script. """ - return 'miso %s' % get_current_args() + return '%s %s' % (Settings.get_miso_exec(), get_current_args()) def strip_option(cmd, option): """ diff --git a/misopy/settings.py b/misopy/settings.py index 8a10b6c..275db70 100644 --- a/misopy/settings.py +++ b/misopy/settings.py @@ -141,6 +141,12 @@ def get_filters(cls, event_type): @classmethod def get(cls): return cls.global_settings + + @classmethod + def get_miso_exec(cls): + if 'MISO_SHELL_EXEC' in os.environ: + return os.environ['MISO_SHELL_EXEC'] + return 'python' def load_settings(settings_filename): diff --git a/misopy/test_cluster.py b/misopy/test_cluster.py index 40cde11..01f95ff 100644 --- a/misopy/test_cluster.py +++ b/misopy/test_cluster.py @@ -4,7 +4,7 @@ ## import unittest import os -import Settings +from misopy.settings import Settings class TestCluster(unittest.TestCase): """ @@ -14,8 +14,8 @@ def setUp(self): # Find out the current directory self.miso_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) self.tests_data_dir = os.path.join(self.miso_path, "test-data") - self.events_analysis_cmd = "miso %s " %(os.path.join(self.miso_path, - "run_events_analysis.py")) + self.events_analysis_cmd = "%s %s " % (Settings.get_miso_exec(), + os.path.join(self.miso_path, "run_events_analysis.py")) self.tests_output_dir = os.path.join(self.miso_path, "test-output") self.test_sam_filename = os.path.join(self.tests_data_dir, "sam-data", @@ -70,7 +70,7 @@ def test_cluster_gene_psi(self): # First index the GFF of interest gff_filename = os.path.join(self.gff_events_dir, "mm9", "genes", "Atp2b1.mm9.gff") gff_index_dir = os.path.join(self.gff_events_dir, "mm9", "indexed") - index_cmd = "miso %s --index %s %s" %(self.index_gff_script, + index_cmd = "%s %s --index %s %s" %(Settings.get_miso_exec(), self.index_gff_script, gff_filename, gff_index_dir) diff --git a/misopy/test_miso.py b/misopy/test_miso.py index 243448a..400a07a 100644 --- a/misopy/test_miso.py +++ b/misopy/test_miso.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import unittest import os +from misopy.settings import Settings class TestMISO(unittest.TestCase): """ @@ -10,8 +11,9 @@ def setUp(self): # Find out the current directory self.miso_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) self.tests_data_dir = os.path.join(self.miso_path, "test-data") - self.events_analysis_cmd = "miso %s " %(os.path.join(self.miso_path, - "run_events_analysis.py")) + self.events_analysis_cmd = "%s %s" % (Settings.get_miso_exec(), + os.path.join(self.miso_path, + "run_events_analysis.py")) self.tests_output_dir = os.path.join(self.miso_path, "test-output") self.test_sam_filename = os.path.join(self.tests_data_dir, "sam-data", @@ -29,9 +31,10 @@ def test_a_sam_to_bam(self): print "Testing conversion of SAM to BAM..." output_dir = os.path.join(self.tests_output_dir, "sam-output") - sam_to_bam_cmd = "miso %s --convert %s %s" %(self.sam_to_bam_script, - self.test_sam_filename, - output_dir) + sam_to_bam_cmd = "%s %s --convert %s %s" % (Settings.get_miso_exec(), + self.sam_to_bam_script, + self.test_sam_filename, + output_dir) print "Executing: %s" %(sam_to_bam_cmd) os.system(sam_to_bam_cmd) @@ -58,9 +61,10 @@ def test_z_gene_psi(self): gff_filename = os.path.join(self.gff_events_dir, "mm9", "genes", "Atp2b1.mm9.gff") gff_index_dir = os.path.join(self.gff_events_dir, "mm9", "genes", "Atp2b1", "indexed") print "Testing GFF indexing of: %s" %(gff_filename) - index_cmd = "python %s --index %s %s" %(self.index_gff_script, - gff_filename, - gff_index_dir) + index_cmd = "%s %s --index %s %s" % (Settings.get_miso_exec(), + self.index_gff_script, + gff_filename, + gff_index_dir) print "Executing: %s" %(index_cmd) os.system(index_cmd) From 76342410e4d1d17623fb6d65d16bbb6d0f83d180 Mon Sep 17 00:00:00 2001 From: Marcus Breese Date: Mon, 30 Jul 2012 01:31:03 -0700 Subject: [PATCH 10/10] changed 'python' to sys.executable (see cassj patch) --- misopy/settings.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/misopy/settings.py b/misopy/settings.py index 275db70..e10cdfc 100644 --- a/misopy/settings.py +++ b/misopy/settings.py @@ -1,10 +1,12 @@ ## ## Settings with relevant directories ## + import misopy from misopy.parse_csv import * import ConfigParser import os +import sys miso_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) @@ -146,7 +148,7 @@ def get(cls): def get_miso_exec(cls): if 'MISO_SHELL_EXEC' in os.environ: return os.environ['MISO_SHELL_EXEC'] - return 'python' + return sys.executable def load_settings(settings_filename):