Skip to content

Commit 9b3409d

Browse files
mmore500mmore500
committed
Add exhaustinve fitness complexity kickoff
1 parent d47b6ce commit 9b3409d

1 file changed

Lines changed: 299 additions & 0 deletions

File tree

‎slurm/competition/fitnesscomplexity-competition-10min-fromwtasphenequivkickoff.sh‎

Lines changed: 299 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,299 @@
1+
#!/bin/bash
2+
3+
################################################################################
4+
echo
5+
echo "running fitnesscomplexity-competition-frombbnopoutkickoff.sh"
6+
echo "-------------------------------------------------------------------------"
7+
################################################################################
8+
9+
# fail on error
10+
set -e
11+
# adapted from https://unix.stackexchange.com/a/504829
12+
printerr() {
13+
echo "Error occurred:"
14+
awk 'NR>L-4 && NR<L+4 { printf "%-5d%3s%s\n",NR,(NR==L?">>>":""),$0 }' L=$1 $0
15+
}
16+
trap 'printerr $LINENO' ERR
17+
18+
if (( "$#" < 6 )); then
19+
echo "USAGE: [bucket] [configpack] [container_tag] [repo_sha] [series] [stints...]"
20+
exit 1
21+
fi
22+
23+
BUCKET="${1}"
24+
echo "BUCKET ${BUCKET}"
25+
shift
26+
27+
CONFIGPACK="${1}"
28+
echo "CONFIGPACK ${CONFIGPACK}"
29+
shift
30+
31+
CONTAINER_TAG="${1}"
32+
echo "CONTAINER_TAG ${CONTAINER_TAG}"
33+
shift
34+
35+
REPO_SHA="${1}"
36+
echo "REPO_SHA ${REPO_SHA}"
37+
shift
38+
39+
SERIES="${1}"
40+
echo "SERIES ${SERIES}"
41+
shift
42+
43+
STINTS="${@}"
44+
echo "STINTS ${STINTS}"
45+
46+
47+
# set up and jump into temporary work directory
48+
cd "$(mktemp -d)"
49+
50+
# curl repro_runner.sh script into to a temporary file
51+
REPRO_RUNNER="$(mktemp)"
52+
curl -o "${REPRO_RUNNER}" "https://raw.githubusercontent.com/mmore500/dishtiny/${REPO_SHA}/script/repro_runner.sh"
53+
chmod +x "${REPRO_RUNNER}"
54+
55+
################################################################################
56+
echo
57+
echo "Run Job with repro_runner.sh"
58+
echo "--------------------------------"
59+
################################################################################
60+
61+
"${REPRO_RUNNER}" \
62+
-p "${BUCKET}" -u mmore500 -s dishtiny \
63+
--repo_sha "${REPO_SHA}" --container_tag "${CONTAINER_TAG}" \
64+
<< REPRO_RUNNER_HEREDOC_EOF
65+
66+
# fail on error
67+
set -e
68+
# adapted from https://unix.stackexchange.com/a/504829
69+
printerr() {
70+
echo "Error occurred:"
71+
awk 'NR>L-4 && NR<L+4 { printf "%-5d%3s%s\n",NR,(NR==L?">>>":""),\$0 }' L=\$1 \$0
72+
}
73+
trap 'printerr \$LINENO' ERR
74+
75+
################################################################################
76+
echo
77+
echo "STINT \${STINT}:"
78+
echo "fitnesscomplexity-competition-frombbnopoutkickoff.sh"
79+
echo "-------------------------------------------------------------------------"
80+
################################################################################
81+
82+
echo "BUCKET ${BUCKET}"
83+
echo "CONFIGPACK ${CONFIGPACK}"
84+
echo "CONTAINER_TAG ${CONTAINER_TAG}"
85+
echo "REPO_SHA ${REPO_SHA}"
86+
echo "SERIES ${SERIES}"
87+
echo "STINTS ${STINTS}"
88+
89+
source ~/.secrets.sh || :
90+
91+
ENDEAVOR="$(( SERIES / 1000 ))"
92+
NUM_STINTS="$( echo ${STINTS} | wc -w )"
93+
94+
echo "ENDEAVOR \${ENDEAVOR}"
95+
echo "NUM_SERIES \${NUM_SERIES}"
96+
97+
for STINT in ${STINTS}; do
98+
99+
echo
100+
echo "STINT \${STINT}:"
101+
echo "Dispatching Fitness Complexity Competitions"
102+
echo "==========================================="
103+
104+
(
105+
cd "\$(mktemp -d)"
106+
pwd
107+
108+
################################################################################
109+
echo
110+
echo "STINT \${STINT}:"
111+
echo "Download wildtype genome"
112+
echo "----------------------------------------------------------"
113+
################################################################################
114+
115+
WILDTYPE_URI="s3://${BUCKET}/endeavor=\${ENDEAVOR}/genomes/stage=0+what=generated/stint=\${STINT}/series=${SERIES}/a=genome+criteria=abundance+morph=wildtype+proc=0+series=${SERIES}+stint=\${STINT}+thread=0+variation=master+ext=.json.gz"
116+
WILDTYPE_GENOME="\$(mktemp -d)/\$(basename \${WILDTYPE_URI})"
117+
118+
echo "downloading \${WILDTYPE_URI} to \${WILDTYPE_GENOME}"
119+
for retry in {1..20}; do
120+
aws s3 cp --quiet \
121+
"\${WILDTYPE_URI}" \
122+
"\${WILDTYPE_GENOME}" \
123+
&& echo " \${WILDTYPE_URI} download success" && break \
124+
|| (echo "retrying \${WILDTYPE_URI} download (\${retry})" && sleep \$((RANDOM % 10)))
125+
if ((\${retry}==20)); then echo "\${WILDTYPE_URI} download fail" && exit 123123; fi
126+
done
127+
128+
################################################################################
129+
echo
130+
echo "STINT \${STINT}:"
131+
echo "Download biotic background phenotype_equivalent_nopout genome"
132+
echo "----------------------------------------------------------"
133+
################################################################################
134+
135+
BB_PE_NOPOUT_URI="s3://${BUCKET}/endeavor=\${ENDEAVOR}/genomes/stage=0+what=generated/stint=\${STINT}/series=${SERIES}/a=genome+criteria=abundance+morph=wildtype+proc=0+series=${SERIES}+stint=\${STINT}+thread=0+variation=master+ext=.json.gz"
136+
PHENOTYPE_EQUIVALENT_GENOME="\$(mktemp -d)/\$(basename \${BB_PE_NOPOUT_URI})"
137+
138+
echo "downloading \${BB_PE_NOPOUT_URI} to \${PHENOTYPE_EQUIVALENT_GENOME}"
139+
for retry in {1..20}; do
140+
aws s3 cp --quiet \
141+
"\${BB_PE_NOPOUT_URI}" \
142+
"\${PHENOTYPE_EQUIVALENT_GENOME}" \
143+
&& echo " \${BB_PE_NOPOUT_URI} download success" && break \
144+
|| (echo "retrying \${BB_PE_NOPOUT_URI} download (\${retry})" && sleep \$((RANDOM % 10)))
145+
if ((\${retry}==20)); then echo "\${BB_PE_NOPOUT_URI} download fail" && exit 123123; fi
146+
done
147+
148+
#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
149+
150+
echo "PHENOTYPE_EQUIVALENT_GENOME \${PHENOTYPE_EQUIVALENT_GENOME}"
151+
echo "WILDTYPE_GENOME \${WILDTYPE_GENOME}"
152+
153+
################################################################################
154+
echo
155+
echo "STINT \${STINT}:"
156+
echo "Generate Nopped Op Variants of phenotype_equivalent_nopout"
157+
echo "----------------------------------------------------------"
158+
################################################################################
159+
160+
python3 "\${REPRO_WORK_DIRECTORY}/dishtiny/script/make_nopped_op_variants.py" "\${WILDTYPE_GENOME}" "\${PHENOTYPE_EQUIVALENT_GENOME}"
161+
162+
################################################################################
163+
echo
164+
echo "STINT \${STINT}:"
165+
echo "Upload Nopped Op Variants of phenotype_equivalent_nopout"
166+
echo "--------------------------------------------------------"
167+
################################################################################
168+
169+
shopt -s nullglob
170+
171+
echo "uploading any genome files..."
172+
for p in *a=genome*morph=wildtype*variation=* ; do
173+
f="\$(basename "\${p}")"
174+
echo "uploading \${p} -> \${f}"
175+
for retry in {1..20}; do
176+
aws s3 cp --quiet \
177+
"\${p}" \
178+
"s3://${BUCKET}/endeavor=\${ENDEAVOR}/genomes-fromwtasphenequivkickoff/stage=2+what=generated/stint=\${STINT}/series=${SERIES}/\${f}" \
179+
&& echo " \$f upload success" && break \
180+
|| (echo "retrying \${f} upload (\${retry})" && sleep \$((RANDOM % 10)))
181+
if ((\${retry}==20)); then echo "\$f upload fail" && exit 123123; fi
182+
done & pids+=(\$!)
183+
184+
# limit to twenty concurrent upload jobs
185+
while (( \$(jobs -p | wc -l) > 20 )); do sleep 1; done
186+
187+
done
188+
189+
# wait on all forked upload jobs
190+
for pid in "\${pids[@]}"; do
191+
# if child process fails, we fail
192+
wait "\${pid}"
193+
done
194+
195+
unset pids
196+
197+
shopt -u nullglob
198+
199+
################################################################################
200+
echo
201+
echo "STINT \${STINT}:"
202+
echo "Get Runscript Template"
203+
echo "----------------------------------------"
204+
################################################################################
205+
206+
# run jinja on template
207+
JOB_TEMPLATE="\$(mktemp)"
208+
echo "JOB_TEMPLATE \${JOB_TEMPLATE}"
209+
210+
for retry in {1..20}; do
211+
curl \
212+
-o "\${JOB_TEMPLATE}" \
213+
"https://raw.githubusercontent.com/mmore500/dishtiny/${REPO_SHA}/slurm/competition/competitionjob.slurm.sh.jinja" \
214+
&& echo " job template curl success" && break \
215+
|| (echo "retrying job template curl (\${retry})" && sleep \$((RANDOM % 10)))
216+
if ((\${retry}==20)); then echo "job template curl fail" && exit 123123; fi
217+
done
218+
219+
################################################################################
220+
echo
221+
echo "STINT \${STINT}:"
222+
echo "Generate Competition Runscripts"
223+
echo "---------------------------------------------"
224+
################################################################################
225+
226+
shopt -s nullglob
227+
228+
wt=\$(basename "\${WILDTYPE_GENOME}")
229+
230+
for variant in *a=genome*morph=wildtype*variation=* ; do
231+
232+
echo "variant \${variant}"
233+
234+
variation="\$(echo "\${variant}" | keyname extract variation)"
235+
echo "variation \${variation}"
236+
237+
JOB_SCRIPT="\$( keyname pack \
238+
--a "wt_vs_variant" --variation "\${variation}" --ext ".slurm.sh" \
239+
)"
240+
241+
j2 --format=yaml -o "\${JOB_SCRIPT}" "\${JOB_TEMPLATE}" << J2_HEREDOC_EOF
242+
bucket: ${BUCKET}
243+
configpack: ${CONFIGPACK}
244+
container_tag: ${CONTAINER_TAG}
245+
repo_sha: ${REPO_SHA}
246+
first_competitor_url: "s3://${BUCKET}/endeavor=\${ENDEAVOR}/genomes/stage=0+what=generated/stint=\${STINT}/series=${SERIES}/\${wt}"
247+
second_competitor_url: "s3://${BUCKET}/endeavor=\${ENDEAVOR}/genomes-fromwtasphenequiv/stage=2+what=generated/stint=\${STINT}/series=${SERIES}/\${variant}"
248+
output_url: "s3://${BUCKET}/endeavor=\${ENDEAVOR}/variant-competitions-10min-fromwtasphenequiv/stage=3+what=generated/stint=\${STINT}/series=${SERIES}/"
249+
replicate: 0
250+
endeavor: \${ENDEAVOR}
251+
stint: \${STINT}
252+
series: ${SERIES}
253+
run_seconds: 600
254+
J2_HEREDOC_EOF
255+
256+
chmod +x "\${JOB_SCRIPT}"
257+
258+
done
259+
260+
shopt -u nullglob
261+
262+
################################################################################
263+
echo
264+
echo "STINT \${STINT}:"
265+
echo "Bundle and Submit Generated Runscripts"
266+
echo "--------------------------------------"
267+
################################################################################
268+
269+
echo "num generated runscripts \$(ls *.slurm.sh | wc -l)"
270+
271+
# uses slurm stoker script, which zips all runscripts in the current directory
272+
# inside itself, then submits itself as a job to gradually feed runscripts onto
273+
# the queue
274+
275+
# if there are any .slurm.sh files
276+
if ls *.slurm.sh 1> /dev/null 2>&1; then
277+
278+
# adapted from https://superuser.com/a/689340
279+
# and https://stackoverflow.com/a/4642975
280+
wget --retry-connrefused --waitretry=1 --read-timeout=20 --timeout=15 -t 10 -qO- https://raw.githubusercontent.com/mmore500/dishtiny/${REPO_SHA}/script/slurm_stoker_containerized_kickoff.sh | bash -s "${BUCKET}" "${CONTAINER_TAG}" "${REPO_SHA}" "variant-competition~configpack%${CONFIGPACK}~series%${SERIES}~stint%\${STINT}"
281+
282+
else
283+
284+
echo "WARNING no variant competition runscripts generated"
285+
echo "skipping slurm_stoker kickoff"
286+
ls
287+
288+
fi
289+
290+
)
291+
done
292+
293+
################################################################################
294+
echo
295+
echo "Done! (SUCCESS)"
296+
echo "---------------"
297+
################################################################################
298+
299+
REPRO_RUNNER_HEREDOC_EOF

0 commit comments

Comments
 (0)