From 7d1653c9060e007033f45c2802958ce3ef677694 Mon Sep 17 00:00:00 2001
From: Arthur Le Bars <arthur.le-bars@sb-roscoff.fr>
Date: Tue, 2 Jun 2020 15:17:17 +0200
Subject: [PATCH] source-data handle
---
.../docker_compose_generator.cpython-38.pyc | Bin 0 -> 2060 bytes
__pycache__/metadata_generator.cpython-38.pyc | Bin 0 -> 781 bytes
__pycache__/table_parser.cpython-38.pyc | Bin 0 -> 2643 bytes
autoload.py | 7 +-
docker_compose_generator.py | 2 +
table_parser.py | 72 +++++++-----------
templates/compose-template.yml | 1 +
templates/stack_template.yml | 2 +-
8 files changed, 36 insertions(+), 48 deletions(-)
create mode 100644 __pycache__/docker_compose_generator.cpython-38.pyc
create mode 100644 __pycache__/metadata_generator.cpython-38.pyc
create mode 100644 __pycache__/table_parser.cpython-38.pyc
diff --git a/__pycache__/docker_compose_generator.cpython-38.pyc b/__pycache__/docker_compose_generator.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..84d509046e11c2024881f7fc2b2a27ec25116d56
GIT binary patch
literal 2060
zcmZ`)&2QX96rZuZw)eA5(xz%@!5|J@iH)Tya6qU^X{FpEK7>Fk70dC=I!?T{JDzP6
zYjfHjxI_qvKOngvapBKEhzs|*a^hd8IPl)s8#Yxst9di;z44p(@q4r1^!ptG<KG|t
z`hLeJ<ZonFhYidQ%<@klj4+y#alO(p)x0q_b_p|>`HC=eVT?^~G3&?>zjJ!nVl8H$
z(Xq|#Lx(xc<t}Tp&Y4AtmULMUB>ff1fZYJe%@xTSTL;O;QgYtDAma`<7i8S!ox>jQ
z20d8&u=Zgch~JO`H-Pm9tT4}SUJ$r(jon&M3Nd_5hPVHLc*tIabafcO?7%EP1i}f%
zI5rq%#w#*5naM0zEsP4*7IVOot2UqI(GeF<^J$Tne3xfjgfbTq1ry7Gd$j!&X88dS
zM&=}_Dck{jLsL^zixIF)_l7NHO>@RoE9Tj(R8Cp&DB-1YC7%{)D7mupnPiC=Qe~Aq
zo%~8f8?3+n+&{>t+@FX?+mE*WGTHaTl<$Y4^y4@T(mZ4V3uWjt?RpT^z6Wqk?fh18
zqTC=zvP1^KJLt5=3ALMF_1^VCZ!N#M61@cm@OvHoAF!E|Lx}yz74J$ajdSw?s&a=c
zDCQ*lMOs>KEQ}w2KBcm?Ag9KhJ|oXxo|<#x+@71~4l^a3K6DWe%({aXJ80p;09~ur
zG5Za<)5zU7<n0w(ox3ZxE&wktJ^|a^xxu<;z<o<TCSQ`T$ammB)Tp;tJ@A*sD4TSS
znsjcROjAXL>g(zS%i__X>H|t<rPoMaQ?HTumlA*M#tkD7JWyaK$}`C`IqV5+OJ$Z)
zC?hYG9dj8SGiBuk&y<xWrBrrU0K=3cH~=N$0&AdJX@1OwXyLwnEE35B@NsAf7bSL*
zO_G$0HDtgHU;)8Y6Re+d8-Q?xa5(@%Xoucv4FDD{mGrH!Sbi5_6D$Y1gcoR7Y{N!k
z>M25jVe#)ZWgqCW8*@{aWAlv?n0wX6wTV9-n&LKC6%G*9(&09t)*(93J38R|c<O4X
z;vUH0MI<HWn$UHKukzjk`pZo24Q*_jES&NnP@N!v?wzH$?*+k&S(w%u(FK#@0TR4s
z<!bE&rdnX4?!wgfyN2tyeJw-#(=f?`p!x|=RJW^_PrSO7SFz_!a^danKJmht%%|{L
zFfRcF#qc~Oc?Q}P;E-goCl9!H31TmN8767CpK@=<JCL#{KlS}Mks#ViGLeK^6XA#Z
zg3EBmd?;*=HwTPQb01LJWxR|;Qs5PpF^Vf^G=f93vg#&TF_WHVhK91_Nx_wMz|#WG
zl~sE*Y5-sL>yt~?7|E(L8ZAAn9@pw6e_iAcn-=sS1Emb3qv1zhu6+tqxGk^T(Wrho
ztB1!2TtNJ}r^~350V;I$YD$$k$@Ag5*np}iHxzLJ?<rSKn#VC@QCCO2kIKOl5znU3
zs$bv<p=^Cps9wle03T|dSjVFstwLd`ETrQei~fIyb=&HG)jhAFtbwCig@9sS2OZ^p
jR>STwJ{<a=%mwtin}ho<?V0O<W&F7Y_N=Qr7uWv)z+onw
literal 0
HcmV?d00001
diff --git a/__pycache__/metadata_generator.cpython-38.pyc b/__pycache__/metadata_generator.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..3f693bd48b005c04c666e0e91a23b55d5e922f8d
GIT binary patch
literal 781
zcmZWmv2GMG5FLAW?>2!$7tsL%35jAUvQm_e2t@)V6_*0htN_cmc9xaA-iz%k5+zmA
z^9xY&OKz$71uDkgTj@~7n&<I&JTq_nX*Syhl=98@udfn--{G(==!0Wa`x-%jAPTBs
zCl!fjTG3MwR3vvGl3Q9Of_#J`{e$tar~_^b#ltbGeTE<*<fy11g5CkXOhk%%Dl*J4
z@p~Vn<AOVWDz!A+^`;KbM+tV&cc}Iy!Zo=DhRq>jT0|NV8$vr3fJy1%q{#fV<E<90
z@mVAF+WM(o$-0%+=iR2F-w{1qudui*s86j_=O&Ee7nmK~{vMUz`c9VT=HT$)u(a*6
z<Vr5Nv1QY6rg|<|XuKTrF)TDt*k4`xoUvB7j<Ls~_4ol~vRPM8?y=q<!)t=^ql9*X
zM*7duaPWitg!k}e{+Zs88|rYbw}ik&zM)<vCue6xVkQ{uv)0MZ`fO!d?a-M=r{YVg
z{Iu@1liEeD1B531%t$0Qa(ap`n<4<2ldBi^+x{<lfpbURns<^hzsp$Hi?s^&L&h%F
zTn#to0p>IF0AbKu5IQj7zxPmM-JIrm^c1P5J$|F=o2J!`PcM0=;?36km@d-DkA~WB
N!#Hi2Mow@&`UjB&vf=;$
literal 0
HcmV?d00001
diff --git a/__pycache__/table_parser.cpython-38.pyc b/__pycache__/table_parser.cpython-38.pyc
new file mode 100644
index 0000000000000000000000000000000000000000..acfef713a99d6e9df6999bde42a8dfc2a24bb97c
GIT binary patch
literal 2643
zcmZuzOOxY760Rz}ZMXY5{TPN>4bTMMoo+A0KmfrKvkSpK4jt?Q0VoEvYFF8oN0vOL
z@^p_wJ~bRW;3vozBDnJp_yK#bE2p_~;4qin%yRckvw;+qs?029Wv24W(kCk`4FcEb
z$NzkJy+z1R$XvZVVD7>we*wY?rx}TsPa09fvxu3e6<OwKN49x7kpoYbxnnnSDfuhm
z7Pr45+&*KGCw%VQXYyxZj{@#;@0>;r(HJ$kFIqg{jdP0<!#8;gBrAf{D+|Kg=MKv2
z_JTyKVs*4uuk+OfEx)^%uPrFp^pJzG!Pn1Ov}tr(d}BfR<{lB7@inmtx~&}|+<JRK
z>K%S<L3aq>2FYf;3)(%t#&^ysg?K+Ao!!R}J2~o5?ejzlm5xOZC08#7%v~7eOF#s{
z+(2F^XI~M>i-{ca;&2yuhkIbv*UgWU<4k;*$Wq9j1@B#6ILGlGjPec;MMkt{${N`V
zT9MeUkV1Nax`uknKSM4sT!RzXfF5<4niZwCCy5$rFF7vF#7^X()D};rZYcC1?x&g1
z?pUZ{!F3=`mB`Dq$hFPWp3*ib5*>`nA~!2LmbOcg^<@B_Ngv4fM6SW($s65aF&5pv
zeC^I_ce-VI+)XlZ4DodbgCx$1gvXG;q`Ta8Vgf&$Jkov~r+KR4cnhqQP=<uiHR_t%
zr?agqc{;pY+QW=rURcBEjY-%Kl#q&Y3iWkEk($C_6&YCzx}bBGAA*j7?#&CGeMPsS
zoZ98Ek`m?B{(?|au~DF~GR@aJx8{s{?~^Zf=N4+XUs+H&0e)H3J@~hgHl9g%6C|yQ
z@zx&H{(X4%**oN^bmc<2@+)ckLfS^DTETj+SFzGBY^`1B*NlEmt&cY9O^9O)#<d0G
z>u2nn@8))8*V|b45Y5gRnLBEiZ>T-6as7;*F^D3cyTCnHxqGC(u^@A=^6KXo1T-Jq
zV{?BVsGC)AM(f}4O~_rq1K1_NSrya)-{Qf7nRy5^`Yn}rD=0vI?Q6D2=E1l0!QINK
zZ&e<80Nl&!h0%+ZQ`vkQ&;<zpl)O&<N&ZDXgN}kYJJQx^Rz_B@d^r2$L>BoV>=$yJ
zsJFv3pG;K<g)_|nf;_~M4SPwR7b-j!p%e)ZUn!@PNg<8eyjH#)g1y^FPH%@`wWG<$
z9U3`O48-Zo3vVC3(mCipq4zY6XV$$K$g78coV5oche8e?3W+$FHH$%c$!PCV>hFJk
zus=T7=YSLNNz)Svl}Db?P_sCG{9m{p+aUO!YB$Zf$W`Yh?M{-MCuJ184?y@pCSwsb
z(D_)L_C#hN75PR0?rJY927@#oXgkgOg|^B^fM#);D(y)z$&#MXPCgw2gseQtwUre;
ziMY^?DtH3GDwXVGztVOw5xGQnq=O6WQIX~byt0kFujDlEVHkFnmP$J^$p-=~id;MB
z{b^*UU|xHwh)r8?QYFUFM_>!>l#>jc@<q<elN6ebn@nIo$P9K3$Pf6~X~<RFz6m61
zEz=O2PN41HFp<WOsAbwrugIYXfx$u!k~}TP(aPm)xjDVB=|j_WmSi1cXqt|)w0Q#r
z<r_c<_30K4)&RP}e7X(2W!u~#y-pj@WwvM=WNnzY;Ktc(=W5TnV)s{l8q@i-?ff1X
z1?_?Zw7%J%TZ)bt^l6JTZv95F+edN_-jL5Dxe26W$v@!yO(bujz%~gp)s<);D?W#M
z(X{{nfj3She2f@V_a}I<M860`GTXiy#j{069T$5pT8m@dh~sg=r*IC$x)sL{rb)Kc
z$PF+f-$n8lB-rDx5X`y6V+xm*E)bvjzJJTe(Ed0{bAZ%~#))v*?h(-8WSEFToKCVr
z3K`;&0B38Hi?S!viNccyN)GToYd(d2COS;p>d^!WbSSdPj2-Zqb8rBs*zCPeheC=_
z6`=_+R6`LKQ-uu!Q?l$8YM7`nEwNclOKc$EM+#LokO}Sx+5bDn^A8}}<D#ETGZop`
zIpikDI_%kTr|IEiMEb1NMc?sU6EdE;`=9TR_ra?}+_ghZrFX6`K^MY$`7#pxzS@T~
zc$tt}C_~Jev`IX{bo<Y8FdYK`J~T%pHVyno(5-~?7~V8Gi9ZI~9SGGs;ksd#pt6+s
z&nEwd1OZ?`u9=$1*O5c`$qo`MSOdD{C;pAEtNnMDmGLfyTfP9q2M}G}t!BIH8^G*R
MpX~y0Ht0?IKl<{v?f?J)
literal 0
HcmV?d00001
diff --git a/autoload.py b/autoload.py
index c26495b..29ff702 100644
--- a/autoload.py
+++ b/autoload.py
@@ -517,7 +517,8 @@ if __name__ == "__main__":
" modify headers (abims), generate blast banks (doesn't commit them: TODO), initialize GGA instance, load the data and run,"
" the main workflow. To update/add data to container, use --update in conjunction to --full (TODO)")
parser.add_argument("--init-instance",
- help="Initialization of galaxy instance. Run first in an empty instance")
+ help="Initialization of galaxy instance. Run first in an empty instance",
+ action="store_true")
parser.add_argument("--load-data",
help="Create src_data directory tree and load its data into the instance")
parser.add_argument("--run-main",
@@ -551,6 +552,7 @@ if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
if str(args.input).endswith(".json"):
+ print("JSON")
input_json = args.input
else:
tp = table_parser.TableParser()
@@ -564,6 +566,7 @@ if __name__ == "__main__":
for json_sp in json_sp_dict:
sp_dict_list.append(json_sp)
+ metadata = {}
for sp_dict in sp_dict_list:
al = Autoload(species_parameters_dictionary=sp_dict, args=args)
al.main_dir = os.path.abspath(args.dir)
@@ -574,7 +577,7 @@ if __name__ == "__main__":
metadata[genus_species_strain_sex]["initialized"] = True
if args.load_data:
logging.info("loading data into galaxy")
- al.load_data_in_galaxy()
+ # al.load_data()
metadata[genus_species_strain_sex]["data_loaded_in_instance"] = True
if args.run_main:
logging.info("running main workflow")
diff --git a/docker_compose_generator.py b/docker_compose_generator.py
index e76198b..c999460 100644
--- a/docker_compose_generator.py
+++ b/docker_compose_generator.py
@@ -17,6 +17,8 @@ TODO: write the whole yml dict from scratch (would allow the script to be more r
more customizable while being clearer (instead of the default yml string or input docker-compose template)
TODO: read json
+
+API master key or galaxy: MASTER_API_KEY: XXXXXXX (alphanum, user prompt/git env variable)
"""
diff --git a/table_parser.py b/table_parser.py
index 0059fdb..9c44cd8 100755
--- a/table_parser.py
+++ b/table_parser.py
@@ -1,4 +1,5 @@
import os
+import sys
import pandas # xlrd required for excel files reading
import numpy
import json
@@ -28,54 +29,35 @@ class TableParser:
def parse_table(self, extension):
if extension == "xls":
pandas_table = pandas.DataFrame(pandas.read_excel(self.table_file))
- pandas_table = pandas_table.replace(numpy.nan, "", regex=True)
- for char in " ,.()-/":
- pandas_table = pandas_table.replace("\\" + char, "_", regex=True)
- pandas_table = pandas_table.replace("\\__", "_", regex=True)
- pandas_table.loc[pandas_table["genome version"] == "", "genome version"] = "1.0"
- pandas_table.loc[pandas_table["ogs version"] == "", "ogs version"] = "1.0"
- pandas_table.loc[pandas_table["version"] == "", "version"] = "1.0"
- pandas_table.loc[pandas_table["date"] == "", "date"] = datetime.today().strftime("%Y-%m-%d")
- with open(os.path.join(self.dir, self.json_file), 'w') as json_file:
- json_file.truncate(0)
- json_content = list()
- for organism in range(0, len(pandas_table.index)):
- organism_dict = pandas_table.iloc[organism].to_dict()
- for k, v in organism_dict.items():
- v = str(v).split(" ")
- v = "_".join(v)
- v = v.replace("__", "_")
- if v.endswith("_"):
- v = v[:-1]
- json_content.append(organism_dict)
- json.dump(json_content, json_file, indent=4)
-
elif extension == "csv":
pandas_table = pandas.DataFrame(pandas.read_csv(self.table_file))
- pandas_table = pandas_table.replace(numpy.nan, "", regex=True)
- for char in " ,.()-/":
- pandas_table = pandas_table.replace("\\" + char, "_", regex=True)
- pandas_table = pandas_table.replace("\\__", "_", regex=True)
- pandas_table.loc[pandas_table["genome version"] == "", "genome version"] = "1.0"
- pandas_table.loc[pandas_table["ogs version"] == "", "ogs version"] = "1.0"
- pandas_table.loc[pandas_table["version"] == "", "version"] = "1.0"
- pandas_table.loc[pandas_table["date"] == "", "date"] = datetime.today().strftime("%Y-%m-%d")
- with open(os.path.join(self.dir, self.json_file), 'w') as json_file:
- json_file.truncate(0)
- json_content = list()
- for organism in range(0, len(pandas_table.index)):
- organism_dict = pandas_table.iloc[organism].to_dict()
- for k, v in organism_dict.items():
- v = str(v).split(" ")
- v = "_".join(v)
- v = v.replace("__", "_")
- if v.endswith("_"):
- v = v[:-1]
- json_content.append(organism_dict)
- json.dump(json_content, json_file, indent=4)
-
else:
- logging.info("input tabulated file doesn't have the correct extension (supported extensions: xls, xlsx, csv)")
+ logging.info("wrong format: input tabulated file cannot be read (supported formats: xls, xlsx, csv)")
+ sys.exit()
+ pandas_table = pandas_table.replace(numpy.nan, "", regex=True)
+
+ for char in " ,.()-/":
+ pandas_table = pandas_table.replace("\\" + char, "_", regex=True)
+ pandas_table = pandas_table.replace("\\__", "_", regex=True)
+ pandas_table.loc[pandas_table["genome version"] == "", "genome version"] = "1.0"
+ pandas_table.loc[pandas_table["ogs version"] == "", "ogs version"] = "1.0"
+ pandas_table.loc[pandas_table["version"] == "", "version"] = "1.0"
+ pandas_table.loc[pandas_table["date"] == "", "date"] = datetime.today().strftime("%Y-%m-%d")
+ with open(os.path.join(self.dir, self.json_file), 'w') as json_file:
+ json_file.truncate(0)
+ json_content = list()
+ for organism in range(0, len(pandas_table.index)):
+ organism_dict = pandas_table.iloc[organism].to_dict()
+ for k, v in organism_dict.items():
+ v = str(v).split(" ")
+ v = "_".join(v)
+ v = v.replace("__", "_")
+ if v.endswith("_"):
+ v = v[:-1]
+ json_content.append(organism_dict)
+ json.dump(json_content, json_file, indent=4)
+
+
def write_json(self, data, filename):
with open(filename, 'w') as f:
diff --git a/templates/compose-template.yml b/templates/compose-template.yml
index ee3e04e..590923c 100755
--- a/templates/compose-template.yml
+++ b/templates/compose-template.yml
@@ -110,6 +110,7 @@ services:
GALAXY_AUTO_UPDATE_DB: 1
GALAXY_AUTO_UPDATE_CONDA: 1
GALAXY_AUTO_UPDATE_TOOLS: "/galaxy-central/tools_1.yaml"
+ MASTER_API_KEY: dev
BLAT_ENABLED: 1
jbrowse:
diff --git a/templates/stack_template.yml b/templates/stack_template.yml
index 68adc74..1844b7c 100755
--- a/templates/stack_template.yml
+++ b/templates/stack_template.yml
@@ -138,7 +138,7 @@ services:
GALAXY_AUTO_UPDATE_TOOLS: "/galaxy-central/tools_1.yaml"
GALAXY_SHARED_DIR: ""
BLAT_ENABLED: 1
- master_api_key: MASTERLOCK
+ MASTER_API_KEY: dev
networks:
- traefik
- genus_species
--
GitLab