Bugfixes3

docker/ACEseqDockerContainer/Dockerfile

@@ -9,18 +9,18 @@ USER roddy
 
 # Install Bioconda
 
-RUN sudo apt-get update; sudo apt-get install -y bzip2 git
+RUN sudo apt-get update; sudo apt-get install -y bzip2 git curl
 
 RUN cd /home/roddy; \
-	echo "Download Anaconda https://repo.continuum.io/archive/Anaconda2-4.4.0-Linux-x86_64.sh"; \
-	wget -q https://repo.continuum.io/archive/Anaconda2-4.4.0-Linux-x86_64.sh && \
+	echo "Download Anaconda"; \
+	wget -q https://repo.continuum.io/miniconda/Miniconda3-4.5.1-Linux-x86_64.sh && \
 	echo "Install Anaconda" && \
-	sudo bash Anaconda2-4.4.0-Linux-x86_64.sh -b && \
-	rm -rf Anaconda2-4.4.0-Linux-x86_64.sh && \
+	sudo bash Miniconda3-4.5.1-Linux-x86_64.sh -b -p /home/roddy/miniconda3 && \
+	rm -rf Miniconda3-4.5.1-Linux-x86_64.sh && \
 	echo Done && \
-	export PATH=/home/roddy/anaconda2/bin/:$PATH
+	export PATH=/home/roddy/miniconda3/bin/:$PATH
 
-ENV PATH=/home/roddy/anaconda2/bin/:$PATH
+ENV PATH=/home/roddy/miniconda3/bin/:$PATH
 
 RUN env; \
 	conda config --add channels r; \
@@ -47,6 +47,7 @@ RUN mkdir -p $BINARIES_FOLDER && cd $BINARIES_FOLDER && \
 	rm -rf $BINARIES_FOLDER/Roddy &&  \
 	git clone https://github.com/eilslabs/Roddy.git &&  \
 	cd Roddy &&  git checkout 2.4.10b &&  mkdir -p plugins_R2.4 &&  \
+	chmod 777 dist/runtime && \
 	ls -l && \
 	ls -l .git/ && \
 #	git submodule add -f  https://github.com/eilslabs/Roddy-Base-Plugin.git dist/plugins/PluginBase && \
@@ -55,7 +56,7 @@ RUN mkdir -p $BINARIES_FOLDER && cd $BINARIES_FOLDER && \
 	git submodule update --init --recursive && \
 	cd $RODDY_BASE_PLUGINS &&  \
 	ls -l && \
-	(cd DefaultPlugin && git checkout 1.2.0) && \
+	(cd DefaultPlugin && git checkout 951fc80) && \
 	(cd PluginBase && git checkout 1.2.0) && \
 	cp -r DefaultPlugin DefaultPlugin_1.2.0 &&  \
 	cp -r PluginBase PluginBase_1.2.0
@@ -65,9 +66,13 @@ RUN mkdir -p $BINARIES_FOLDER && cd $BINARIES_FOLDER && \
 RUN cd $RODDY_PLUGINS_R2_4 && \
 	git  clone https://github.com/eilslabs/ACEseqWorkflow ACEseqWorkflow_1.2.10 && \
 	cd ACEseqWorkflow_1.2.10  &&  \
-	git checkout 1.2.10
+	git checkout 1.2.10 && \
+	sed -i -e 's/[ \t]\${EMAIL}[ \t]/ "${EMAIL}" /g' $ACESEQ_FOLDER/resources/analysisTools/copyNumberEstimationWorkflow/correct_gc_bias*.sh # wrap variable in quotes, or else e-mail is sent, even when $EMAIL is empty
 
-RUN	conda env create -n ACEseqWorkflow -f $ACESEQ_FOLDER/resources/analysisTools/copyNumberEstimationWorkflow/environments/conda.yml;
+RUN	sed -i -e 's/.*gsl.*//' $ACESEQ_FOLDER/resources/analysisTools/copyNumberEstimationWorkflow/environments/conda.yml && \
+	conda env create -n ACEseqWorkflow -f $ACESEQ_FOLDER/resources/analysisTools/copyNumberEstimationWorkflow/environments/conda.yml && \
+	conda install -n ACEseqWorkflow -y bedtools=2.24.0=0 blas=1.1=openblas libgcc-ng=7.2.0=hdf63c60_3 libgfortran=3.0.0=1 libstdcxx-ng=7.2.0=hdf63c60_3 numpy=1.11.3=py27_blas_openblashd3ea46f_205 openblas=0.2.20=8 gsl=2.4=blas_openblas_0 && \
+	conda clean --all -y
 
 # Install Groovy 2.4.7 from Maven. OpenJDK is already installed in the base image so just link it..
 RUN mkdir -p ~/.roddy/runtimeDevel && cd ~/.roddy/runtimeDevel && \
@@ -76,7 +81,8 @@ RUN mkdir -p ~/.roddy/runtimeDevel && cd ~/.roddy/runtimeDevel && \
 	wget $groovyZipLink && \
 	unzip $groovyArchive && \
 	rm $groovyArchive && \
-	ln -sf /usr/lib/jvm/java-8-openjdk-amd64 ~/.roddy/runtimeDevel/jdk && ln -sf ~/.roddy/runtimeDevel/jdk ~/.roddy/runtimeDevel/jdk1.8.0_131
+	ln -sf /usr/lib/jvm/java-8-openjdk-amd64 ~/.roddy/runtimeDevel/jdk && ln -sf ~/.roddy/runtimeDevel/jdk ~/.roddy/runtimeDevel/jdk1.8.0_131 && \
+	mv ~/.roddy ~/writable.template/.roddy && ln -s /home/roddy/writable/.roddy /home/roddy/.roddy # required for Singularity (will be copied to writable dir and symlinked)
 
 # Currently not needed. Both are packed in R2.4.10
 #RUN cd $BINARIES_FOLDER && \
@@ -99,3 +105,9 @@ RUN cd $RODDY_PLUGINS_R2_4 && git clone https://github.com/eilslabs/COWorkflowsB
 ## HINT: 				 ~/binaries/Roddy/roddy.sh run coWorkflowsTestProject@aceseq stds --c=/home/roddy/config/ini/alllocal.ini
 ## HINT: Run the container with: docker run -v /home/heinold/.roddy/runtimeDevel/:/home/roddy/.roddy/runtimeDevel -v /home/heinold/temp:/home/roddy/temp -v `readlink -f config`:/home/roddy/config  -t -i aceseqimage /bin/bash
 
+# copy workflow config
+COPY config /home/roddy/config
+
+# create an empty DELLY SV file, for when the workflow is called without SVs
+RUN echo -e 'chrom1\tstart1\tend1\tchrom2\tstart2\tend2\tid\tpairs\tstrand1\tstrand2\tsvtype\tsize\torient\tmapq\tsplit_reads\tsplit_mapq\tsplit_consensus\tpid\taf\tgenotypes\trd_ratio\ttumor_count\tgerm_count\tBf\tdepth.ratio\tCNt\tA\tB\tCN_supp' > /home/roddy/empty_sv.tsv
+

docker/ACEseqDockerContainer/config/ini/alllocal.ini

@@ -2,7 +2,7 @@
 useRoddyVersion=current
 
 [DIRECTORIES]
-configurationDirectories=/home/roddy/config/xml,/home/roddy/additionalConfigs
+configurationDirectories=/home/roddy/config/xml
 pluginDirectories=/home/roddy/binaries/Roddy/plugins_R2.4
 
 [COMMANDS]

docker/ACEseqDockerContainer/config/xml/projectsACEseqTest.xml

@@ -1,4 +1,4 @@
-<configuration configurationType='project' name='ACEseq' imports='sampleNames' description='' >
+<configuration configurationType='project' name='ACEseq' description='' >
 
   <availableAnalyses>
     <analysis id="copyNumberEstimation" configuration="copyNumberEstimationAnalysis"
@@ -7,12 +7,13 @@
 
   <configurationvalues>
    <cvalue name="workflowEnvironmentScript" value="${TOOL_WORKFLOW_ENVIRONMENT_CONDA}" type="string"/>
-   <cvalue name="condaEnvironmentName" value="ACEseqWorkflow" type="string"
+   <cvalue name="condaEnvironmentName" value="/home/roddy/.conda/envs/ACEseqWorkflow" type="string"
                 description="Name of the Conda environment on the execution hosts. Used by the environment setup script in resources/analysisTools/copyNumberEstimation/enviroments."/>
 
     <cvalue name='inputBaseDirectory' value='pathToPidStructure/' type='path'/>
     <cvalue name='outputBaseDirectory'  value='pathToPidStructure/' type='path'/>
     <cvalue name='outputAnalysisBaseDirectory' value='${outputBaseDirectory}/${pid}' type='path'/>
+    <cvalue name='aceseqOutputDirectory' value='${outputAnalysisBaseDirectory}/ACEseq' type="path"/>
 
     <cvalue name='outputAllowAccessRightsModification' value='false' type="boolean"/>
 
@@ -23,12 +24,11 @@
 -->
   </configurationvalues>
 
-<!--Adjust this to the filename of your SV bedpe with SV points
       <filenames package='de.dkfz.b080.co.files' filestagesbase='de.dkfz.b080.co.files.COFileStage'>
-        <filename class="TextFile" onMethod="de.dkfz.b080.co.aceseq.ACESeqMethods.mergeDelly"
-                  selectiontag="dellyFileTag"
-                  pattern='${svOutputDirectory}/${pid}_svs.bedpe'/>
+        <filename class="TextFile" onMethod="de.dkfz.b080.co.aceseq.ACESeqMethods.mergeSv"
+                  selectiontag="svFileTag"
+                  pattern='${svFile}'/>
       </filenames>
--->
+
 
 </configuration>

docker/ACEseqDockerContainer/roddy.sh

@@ -1,36 +1,36 @@
 #!/bin/bash
 # 1: Run mode, which might be "run" or "testrun"
-# 2: Configuration identifier, normally "ACEseq"
-# 3: Configuration directory
-# 4: Dataset identifier / PID
-# 5: Control bam file
-# 6: Tumor bam file
-# 7: Control bam sample name
-# 8: Tumor bam sample name
-# 9: Reference genome file
-# 10: Reference files path
-# 11: Output folder
+# 2: Container type, "docker" or "singularity"
+# 3: Dataset identifier / PID
+# 4: Control bam file
+# 5: Tumor bam file
+# 6: Control bam sample name
+# 7: Tumor bam sample name
+# 8: Reference genome file
+# 9: Reference files path
+# 10: Output folder
+# 11: Number of threads
 # 12: Optional: The SV file
 
 if [[ $# -lt 11 ]]; then
 	echo "Wrong number of arguments"
-	head -n 12 "$0" | tail -n+2
+	head -n 13 "$0" | tail -n+2
 	exit 1
 fi
 
 ## Read in parameters and check files and folders
 mode=${1}
-configurationIdentifier=${2}
-configurationFolderLcl=`readlink -f "${3}"`
-pid=${4}
+container=${2}
+pid=${3}
 
-inputBamCtrlLcl=`readlink -f "${5}"`
-inputBamTumorLcl=`readlink -f "${6}"`
-inputBamCtrlSampleName=${7}
-inputBamTumorSampleName=${8}
-referenceGenomeFile=`readlink -f "${9}"`
-referenceFilesPath=`readlink -f "${10}"`
-workspaceLcl=`readlink -f "${11}"`
+inputBamCtrl=`readlink -m "${4}"`
+inputBamTumor=`readlink -m "${5}"`
+inputBamCtrlSampleName=${6}
+inputBamTumorSampleName=${7}
+referenceGenomeFile=`readlink -m "${8}"`
+referenceFilesPath=`readlink -m "${9}"`
+workspace=`readlink -m "${10}"`
+threads=${11}
 
 function checkFile() {
 	local _file=${1}
@@ -49,57 +49,67 @@ function checkDir() {
 }
 
 [[ $mode -ne "run" && $mode -ne "testrun" ]] && echo "Mode must be run or testrun" && exit 2
-checkFile $inputBamCtrlLcl 
-checkFile $inputBamTumorLcl 
+[ "$container" != "docker" -a "$container" != "singularity" ] && echo "Container must be docker or singularity" && exit 2
+checkFile $inputBamCtrl
+checkFile $inputBamTumor
 checkFile ${referenceGenomeFile}
+checkFile ${referenceGenomeFile}.fai
 checkDir $referenceFilesPath 
-checkDir $workspaceLcl rw
+checkDir $workspace rw
+! [[ $threads =~ ^[1-9]|1[0-9]|2[0-4]*$ ]] && echo "Number of threads must be between 1 and 24" && exit 2
 
 if [[ $# -eq 12 ]]; then
-	# Either use the file
-	local svFile=`readlink -f ${12}`
+	# Either use the provided SV file
+	svFile=`readlink -m ${12}`
 	checkFile $svFile
 	svBlock="svFile:${svFile}"
 else 
-	# or explicitely disable it.
-	svBlock="runWithSv:false,SV:no"
+	# or use an empty SV file to disable it.
+	svBlock="svFile:/home/roddy/empty_sv.tsv"
 fi
 
 # Define in-Docker files and folders
-
-workspace=/home/roddy/workspace
-configurationFolder=/home/roddy/config
-inputBamCtrl=${inputBamCtrlLcl}
-inputBamTumor=${inputBamTumorLcl}
-
 roddyBinary="bash /home/roddy/binaries/Roddy/roddy.sh"
 roddyConfig="--useconfig=/home/roddy/config/ini/alllocal.ini"
 bamFiles="bamfile_list:$inputBamCtrl;$inputBamTumor"
 sampleList="sample_list:${inputBamCtrlSampleName};${inputBamTumorSampleName}"
+sampleListParameters="possibleTumorSampleNamePrefixes:(${inputBamTumorSampleName}),possibleControlSampleNamePrefixes:(${inputBamCtrlSampleName})"
 tumorSample="tumorSample:${inputBamTumorSampleName}"
 baseDirectoryReference="baseDirectoryReference:${referenceFilesPath}"
 referenceGenome="REFERENCE_GENOME:${referenceGenomeFile}"
-referenceGenomePath=`dirname ${referenceGenomeFile}`
 outputBaseDirectory="outputBaseDirectory:${workspace}"
 outputFileGroup="outputFileGroup:roddy"
-sampleListParameters="possibleTumorSampleNamePrefixes:(${inputBamTumorSampleName}),possibleControlSampleNamePrefixes:(${inputBamCtrlSampleName})"
 
-sampleXML=/home/roddy/additionalConfigs/sampleNames.xml
-prepareAdditionalConfigCall="mkdir /home/roddy/additionalConfigs && cp /home/roddy/config/xml/sampleConfigTemplate.txt $sampleXML && sed -i  -e 's/CONTROL_SAMPLE/${inputBamCtrlSampleName}/' $sampleXML && sed -i  -e 's/TUMOR_SAMPLE/${inputBamTumorSampleName}/' $sampleXML "
+call="${roddyBinary} ${mode} ACEseq@copyNumberEstimation ${pid} ${roddyConfig} --cvalues='${bamFiles},${svBlock},${sampleList},${sampleListParameters},${tumorSample},${referenceGenome},${baseDirectoryReference},${outputBaseDirectory},${outputFileGroup}'"
 
-call="${roddyBinary} ${mode} ${configurationIdentifier}@copyNumberEstimation ${pid} ${roddyConfig} --cvalues=\"${bamFiles},${svBlock},${sampleList},${tumorSample},${referenceGenome},${baseDirectoryReference},${outputBaseDirectory},${outputFileGroup}\""
+mkdir -p "${workspace}/${pid}"
 
-absoluteCall="[[ ! -d ${workspace}/${pid} ]] && mkdir ${workspace}/${pid}; $prepareAdditionalConfigCall ;$call; echo \"Wait for Roddy to finish\"; "'while [[ 2 -lt $(qstat | wc -l ) ]]; do echo $(expr $(qstat | wc -l) - 2 )\" jobs are still in the list\"; sleep 120; done;'" echo \"done\"; ec=$?"
-
-docker run \
-		-v ${inputBamCtrlLcl}:${inputBamCtrl} -v ${inputBamCtrlLcl}.bai:${inputBamCtrl}.bai \
-		-v ${inputBamTumorLcl}:${inputBamTumor} -v ${inputBamTumorLcl}.bai:${inputBamTumor}.bai \
-		-v ${workspaceLcl}:${workspace} \
-		-v ${referenceGenomePath}:${referenceGenomePath} \
-		-v "${referenceFilesPath}:${referenceFilesPath}" \
-		-v "${configurationFolderLcl}:${configurationFolder}" \
+if [ "$container" = "docker" ]; then
+	docker run \
+		-v "${inputBamCtrl}:${inputBamCtrl}:ro" -v "${inputBamCtrl}.bai:${inputBamCtrl}.bai:ro" \
+		-v "${inputBamTumor}:${inputBamTumor}:ro" -v "${inputBamTumor}.bai:${inputBamTumor}.bai:ro" \
+		-v "${workspace}:${workspace}" \
+		-v "${referenceGenomeFile}:${referenceGenomeFile}:ro" -v "${referenceGenomeFile}.fai:${referenceGenomeFile}.fai:ro" \
+		-v "${referenceFilesPath}:${referenceFilesPath}:ro" \
+		$([ -n "${svFile}" ] && echo -v "${svFile}:${svFile}:ro") \
 		--rm \
-		--user 0 --env=RUN_AS_UID=`id -u` --env=RUN_AS_GID=`id -g` \
-		-t -i aceseqimage \
-		/bin/bash -c "$absoluteCall"
+		--shm-size=1G \
+		--user $(id -u):$(id -g) \
+		--env=THREADS=$threads \
+		aceseqimage \
+		/bin/bash -c "$call"
+else
+	export SINGULARITYENV_THREADS=$threads
+	singularity exec \
+		-B /tmp:/tmp \
+		-B "${inputBamCtrl}:${inputBamCtrl}:ro" -B "${inputBamCtrl}.bai:${inputBamCtrl}.bai:ro" \
+		-B "${inputBamTumor}:${inputBamTumor}:ro" -B "${inputBamTumor}.bai:${inputBamTumor}.bai:ro" \
+		-B "${workspace}:${workspace}" \
+		-B "${referenceGenomeFile}:${referenceGenomeFile}:ro" -B "${referenceGenomeFile}.fai:${referenceGenomeFile}.fai:ro" \
+		-B "${referenceFilesPath}:${referenceFilesPath}:ro" \
+		$([ -n "${svFile}" ] && echo -B "${svFile}:${svFile}:ro") \
+		--containall --net \
+		$(dirname "$0")/singularity.img \
+		/ENTRYPOINT.sh /bin/bash -c "$call"
+fi
 

docker/BaseDockerContainer/Dockerfile

@@ -19,7 +19,8 @@ RUN apt-get update && \
 					   libgfortran3 libglu1-mesa-dev wget libfreetype6 libfreetype6-dev libpng-dev libcurl4-openssl-dev \
 					   gfortran libcairo2 libjpeg-dev ghostscript vim sudo bsdmainutils aptitude \
 					   gtk2-engines libxtst6 libxxf86vm1 freeglut3 libxslt1.1 \
-					   python-gi-cairo openjdk-8-jdk gosu;
+					   python-gi-cairo openjdk-8-jdk; \
+    apt-get clean
 
 RUN cpanm XML::Parser --notest &&  \
     cpanm Math::CDF --notest
@@ -28,58 +29,45 @@ RUN	pip install python-dateutil && \
     easy_install -U 'distribute';
 
 
-# Install the SGE
+# import scripts to install gridengine
 ADD scripts/gridEngineSetup.sh $binaries/gridEngineSetup.sh
-
 ADD scripts/gridEngineConfiguration.txt $binaries/gridEngineConfiguration.txt
 
-RUN	[ "/bin/bash" , "-c" , "ls -l $binaries; source /home/roddy/binaries/gridEngineSetup.sh; setupHost; installGridEngine; setupGridEngine;"  ]
-
+# Install gridengine.
+# GridEngine writes to multiple directoriers, but Singularity requires images to be read-only.
+# => Move all writable directories aside.
+# At runtime we mount a scratch dir from the host and copy these directories there.
+# Using a symlink we point to the new location in the scratch dir.
+RUN bash -c "source /home/roddy/binaries/gridEngineSetup.sh; setupHost; installGridEngine; setupGridEngine" && \
+    mkdir /home/roddy/writable.template && chmod a+rwx /home/roddy/writable.template && \
+    mv /var/spool/gridengine /home/roddy/writable.template/var_spool_gridengine && \
+    ln -s /home/roddy/writable/var_spool_gridengine /var/spool/gridengine && \
+    mv /etc/gridengine/bootstrap /home/roddy/writable.template/etc_gridengine_bootstrap && \
+    ln -s /home/roddy/writable/etc_gridengine_bootstrap /etc/gridengine/bootstrap && \
+    mv /var/lib/gridengine/default/common /home/roddy/writable.template/var_lib_gridengine_default_common && \
+    ln -s /home/roddy/writable/var_lib_gridengine_default_common /var/lib/gridengine/default/common && \
+    mv /var/run/gridengine /home/roddy/writable.template/var_run_gridengine && \
+    ln -s /home/roddy/writable/var_run_gridengine /var/run/gridengine
 
 # Create all kinds of directories for Roddy and projects
 # Add the Roddy user and bash things
-RUN mkdir -p /workspace/output && \
-    mkdir -p /workspace/input && \
-    mkdir -p /home/roddy/.roddy/runtimeDevel && \
-    mkdir -p /home/roddy/binaries && \
+RUN mkdir -p /home/roddy/binaries && \
     mkdir -p /data/roddyScratch && \
-    mkdir -p /icgc/ngs_share && \
     useradd roddy -d /home/roddy && \
-    chown -R roddy:roddy /workspace && \
     chown -R roddy:roddy /home/roddy && \
     chown -R roddy:roddy /data/roddyScratch && \
-    chown -R roddy:roddy /icgc && \
     chmod -R 777 /data/roddyScratch && \
+    chmod 777 /home/roddy /dev/stdout /dev/stderr && \
     adduser roddy sudo && \
     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 
-USER roddy
-
-ADD BASHRC /home/roddy/.bashrc
-
-#ADD commondependencies /home/roddy/temporary
-
-#RUN sudo chown -R roddy:roddy /home/roddy/temporary && \
-#    sudo chmod -R 777 /home/roddy/temporary && \
-#    sudo chmod -R 777 /icgc && \
-#    cd /home/roddy/temporary && \
-#    (unzip Roddy.zip && rm Roddy.zip && mv Roddy $binaries) && \
-#    (unzip `ls apache-groovy*zip` && rm apache-groovy*.zip && mv groovy* $runtime) && \
-#    sudo chmod -R +x /home/roddy/binaries/Roddy/dist/plugins
-
-#    rm -rf /home/roddy/temporary && \
-#    (tar -xvz --file=`ls j*` && rm j*.gz && mv j* $runtime) && \
-
 ADD ENTRYPOINT.sh /ENTRYPOINT.sh
-
-USER root
-
 RUN chmod 777 /ENTRYPOINT.sh
+ENTRYPOINT ["/ENTRYPOINT.sh"]
 
-USER roddy
+# passwd and group files need to be updated at runtime to give roddy the same ID as the calling user
+RUN chmod a+rw /etc /etc/passwd /etc/group && \
+    sed -i -e 's|/root|/home/roddy|' /etc/passwd # give root the same home as the roddy user to give both users the same environment
 
-ENTRYPOINT ["/ENTRYPOINT.sh"]
+ADD BASHRC /home/roddy/.bashrc
 
-#USER roddy
-#RUN sudo bash -c "echo '127.0.0.1 master' | cat - /etc/hosts > /tmp/tmp_host && cp /tmp/tmp_host /etc/hosts"; \
-#    cat /etc/hosts;