Release 2.0 (merge dev to master)

General

• Reorganize repository
• Fix verbosity mode
• Fix logs details
• Update README.md
• Update the "help" option in the scripts
• Update examples
• Factorize and optimize code
• Create file constants.py containing the constant variables

Input organisms.yml file

• Change root structure (a list instead of a dictionary, the name parameter is added to the species dictionary sp_dict)
• Add parameter main_species. Useful when there are multiple strains for the same species: set to yes to define the strain used as home for JBrowse
• Parameters species, genus, strain, sex, common will be sanitized (removing " ", "-", "(", ")", "'")
• Add jinja2 template

Input config file

• Change root structure (one-level dictionary)
• Add parameters authelia_secrets_env_path and authelia_db_postgres_password required for Authelia
• Change the name of some parameters (galaxy_persist_data, tripal_banner_path, jbrowse_menu_url)
• Remove unused apollo parameters (webapollo_user, webapollo_password)

Step "init"

• Update templates files
• Fix the copy of the specified banner
• Write galaxy_nginx.conf only if needed

Step "get data"

• Prefix data filenames with g_species_s
• Add the last modification date of the source file to the gff filename

Step "load data"

• Use Bioblend function upload_from_galaxy_fs() instead of upload_from_local() (to enable symlink, instead of copy)

Step "run workflow Phaeoexplorer"

• Development of a workflow loading fasta and gff files to Chado, Tripal and JBrowse, for one or two organisms of the same species

workflows_phaeoexplorer/Galaxy-Workflow-load_blast_results_2org_v1.ga

@@ -20,14 +20,14 @@
             "name": "Input dataset",
             "outputs": [],
             "position": {
-                "bottom": 356.1999969482422,
-                "height": 82.19999694824219,
-                "left": 168,
-                "right": 368,
-                "top": 274,
+                "bottom": 230.39999389648438,
+                "height": 61.19999694824219,
+                "left": 97.5,
+                "right": 297.5,
+                "top": 169.1999969482422,
                 "width": 200,
-                "x": 168,
-                "y": 274
+                "x": 97.5,
+                "y": 169.1999969482422
             },
             "tool_id": null,
             "tool_state": "{\"optional\": false}",
@@ -38,7 +38,7 @@
                 {
                     "label": null,
                     "output_name": "output",
-                    "uuid": "8c6e3cd8-d0a0-403f-a882-f5206c8627f9"
+                    "uuid": "72006174-6297-4777-95bd-ca427b9ea729"
                 }
             ]
         },
@@ -58,14 +58,14 @@
             "name": "Input dataset",
             "outputs": [],
             "position": {
-                "bottom": 467.1999969482422,
-                "height": 82.19999694824219,
-                "left": 200,
-                "right": 400,
-                "top": 385,
+                "bottom": 341.40000915527344,
+                "height": 61.19999694824219,
+                "left": 129.5,
+                "right": 329.5,
+                "top": 280.20001220703125,
                 "width": 200,
-                "x": 200,
-                "y": 385
+                "x": 129.5,
+                "y": 280.20001220703125
             },
             "tool_id": null,
             "tool_state": "{\"optional\": false}",
@@ -76,7 +76,7 @@
                 {
                     "label": null,
                     "output_name": "output",
-                    "uuid": "8d7cef33-5fdc-481b-a0fd-542c42c6d032"
+                    "uuid": "45971e82-4e85-4993-a9cb-9a4608e9def7"
                 }
             ]
         },
@@ -96,10 +96,6 @@
                     "description": "runtime parameter for tool Chado load Blast results",
                     "name": "analysis_id"
                 },
-                {
-                    "description": "runtime parameter for tool Chado load Blast results",
-                    "name": "input"
-                },
                 {
                     "description": "runtime parameter for tool Chado load Blast results",
                     "name": "organism_id"
@@ -118,14 +114,14 @@
                 }
             ],
             "position": {
-                "bottom": 361.3999938964844,
-                "height": 164.39999389648438,
-                "left": 528,
-                "right": 728,
-                "top": 197,
+                "bottom": 255.8000030517578,
+                "height": 163.60000610351562,
+                "left": 457.5,
+                "right": 657.5,
+                "top": 92.19999694824219,
                 "width": 200,
-                "x": 528,
-                "y": 197
+                "x": 457.5,
+                "y": 92.19999694824219
             },
             "post_job_actions": {},
             "tool_id": "toolshed.g2.bx.psu.edu/repos/gga/chado_load_blast/load_blast/2.3.4+galaxy0",
@@ -135,7 +131,7 @@
                 "owner": "gga",
                 "tool_shed": "toolshed.g2.bx.psu.edu"
             },
-            "tool_state": "{\"analysis_id\": {\"__class__\": \"RuntimeValue\"}, \"blastdb_id\": \"21\", \"input\": {\"__class__\": \"RuntimeValue\"}, \"match_on_name\": \"false\", \"organism_id\": {\"__class__\": \"RuntimeValue\"}, \"psql_target\": {\"method\": \"remote\", \"__current_case__\": 0}, \"query_type\": \"polypeptide\", \"re_name\": \"\", \"skip_missing\": \"false\", \"wait_for\": {\"__class__\": \"RuntimeValue\"}, \"__page__\": null, \"__rerun_remap_job_id__\": null}",
+            "tool_state": "{\"analysis_id\": {\"__class__\": \"RuntimeValue\"}, \"blastdb_id\": \"21\", \"input\": {\"__class__\": \"ConnectedValue\"}, \"match_on_name\": \"false\", \"organism_id\": {\"__class__\": \"RuntimeValue\"}, \"psql_target\": {\"method\": \"remote\", \"__current_case__\": 0}, \"query_type\": \"polypeptide\", \"re_name\": \"\", \"skip_missing\": \"false\", \"wait_for\": {\"__class__\": \"RuntimeValue\"}, \"__page__\": null, \"__rerun_remap_job_id__\": null}",
             "tool_version": "2.3.4+galaxy0",
             "type": "tool",
             "uuid": "595f6e1f-955a-42be-b03b-1269d1f7d189",
@@ -143,33 +139,33 @@
                 {
                     "label": null,
                     "output_name": "results",
-                    "uuid": "c0ad63b8-045e-4a7c-97ba-8bc45862550b"
+                    "uuid": "cb238779-29f4-4f22-b6f3-6a8cc84857d1"
                 }
             ]
         },
         "3": {
             "annotation": "",
-            "content_id": "toolshed.g2.bx.psu.edu/repos/gga/chado_load_blast/load_blast/2.3.4+galaxy0",
+            "content_id": "toolshed.g2.bx.psu.edu/repos/gga/tripal_analysis_sync/analysis_sync/3.2.1.0",
             "errors": null,
             "id": 3,
             "input_connections": {
-                "input": {
-                    "id": 1,
-                    "output_name": "output"
+                "wait_for": {
+                    "id": 2,
+                    "output_name": "results"
                 }
             },
             "inputs": [
                 {
-                    "description": "runtime parameter for tool Chado load Blast results",
-                    "name": "input"
+                    "description": "runtime parameter for tool Synchronize an analysis",
+                    "name": "analysis_id"
                 },
                 {
-                    "description": "runtime parameter for tool Chado load Blast results",
+                    "description": "runtime parameter for tool Synchronize an analysis",
                     "name": "wait_for"
                 }
             ],
-            "label": "load blast results org2",
-            "name": "Chado load Blast results",
+            "label": "sync blast analysis org1",
+            "name": "Synchronize an analysis",
             "outputs": [
                 {
                     "name": "results",
@@ -177,54 +173,70 @@
                 }
             ],
             "position": {
-                "bottom": 545.3999938964844,
-                "height": 164.39999389648438,
-                "left": 591,
-                "right": 791,
-                "top": 381,
+                "bottom": 254.8000030517578,
+                "height": 153.60000610351562,
+                "left": 787.5,
+                "right": 987.5,
+                "top": 101.19999694824219,
                 "width": 200,
-                "x": 591,
-                "y": 381
+                "x": 787.5,
+                "y": 101.19999694824219
             },
             "post_job_actions": {},
-            "tool_id": "toolshed.g2.bx.psu.edu/repos/gga/chado_load_blast/load_blast/2.3.4+galaxy0",
+            "tool_id": "toolshed.g2.bx.psu.edu/repos/gga/tripal_analysis_sync/analysis_sync/3.2.1.0",
             "tool_shed_repository": {
-                "changeset_revision": "08ae8b27b193",
-                "name": "chado_load_blast",
+                "changeset_revision": "f487ff676088",
+                "name": "tripal_analysis_sync",
                 "owner": "gga",
                 "tool_shed": "toolshed.g2.bx.psu.edu"
             },
-            "tool_state": "{\"analysis_id\": \"1\", \"blastdb_id\": \"1\", \"input\": {\"__class__\": \"RuntimeValue\"}, \"match_on_name\": \"false\", \"organism_id\": \"2\", \"psql_target\": {\"method\": \"remote\", \"__current_case__\": 0}, \"query_type\": \"polypeptide\", \"re_name\": \"\", \"skip_missing\": \"false\", \"wait_for\": {\"__class__\": \"RuntimeValue\"}, \"__page__\": null, \"__rerun_remap_job_id__\": null}",
-            "tool_version": "2.3.4+galaxy0",
+            "tool_state": "{\"analysis_id\": {\"__class__\": \"RuntimeValue\"}, \"wait_for\": {\"__class__\": \"RuntimeValue\"}, \"__page__\": null, \"__rerun_remap_job_id__\": null}",
+            "tool_version": "3.2.1.0",
             "type": "tool",
-            "uuid": "5d08ed84-681b-431c-92c9-3b2fd5f32176",
+            "uuid": "c98dedf6-8857-4d23-be94-fe6630f245d7",
             "workflow_outputs": [
                 {
-                    "label": null,
+                    "label": "Synchronize Analysis into Tripal",
                     "output_name": "results",
-                    "uuid": "6ed0a7c4-5b78-4016-8c21-0aaa992acd11"
+                    "uuid": "1ff4b1db-b6bf-4c48-a0ab-0a8513683999"
                 }
             ]
         },
         "4": {
             "annotation": "",
-            "content_id": "toolshed.g2.bx.psu.edu/repos/gga/tripal_analysis_sync/analysis_sync/3.2.1.0",
+            "content_id": "toolshed.g2.bx.psu.edu/repos/gga/chado_load_blast/load_blast/2.3.4+galaxy0",
             "errors": null,
             "id": 4,
             "input_connections": {
+                "input": {
+                    "id": 1,
+                    "output_name": "output"
+                },
                 "wait_for": {
-                    "id": 2,
+                    "id": 3,
                     "output_name": "results"
                 }
             },
             "inputs": [
                 {
-                    "description": "runtime parameter for tool Synchronize an analysis",
+                    "description": "runtime parameter for tool Chado load Blast results",
                     "name": "analysis_id"
+                },
+                {
+                    "description": "runtime parameter for tool Chado load Blast results",
+                    "name": "input"
+                },
+                {
+                    "description": "runtime parameter for tool Chado load Blast results",
+                    "name": "organism_id"
+                },
+                {
+                    "description": "runtime parameter for tool Chado load Blast results",
+                    "name": "wait_for"
                 }
             ],
-            "label": "sync blast analysis org1",
-            "name": "Synchronize an analysis",
+            "label": "load blast results org2",
+            "name": "Chado load Blast results",
             "outputs": [
                 {
                     "name": "results",
@@ -232,32 +244,32 @@
                 }
             ],
             "position": {
-                "bottom": 360.3999938964844,
-                "height": 154.39999389648438,
-                "left": 858,
-                "right": 1058,
-                "top": 206,
+                "bottom": 439.8000183105469,
+                "height": 163.60000610351562,
+                "left": 520.5,
+                "right": 720.5,
+                "top": 276.20001220703125,
                 "width": 200,
-                "x": 858,
-                "y": 206
+                "x": 520.5,
+                "y": 276.20001220703125
             },
             "post_job_actions": {},
-            "tool_id": "toolshed.g2.bx.psu.edu/repos/gga/tripal_analysis_sync/analysis_sync/3.2.1.0",
+            "tool_id": "toolshed.g2.bx.psu.edu/repos/gga/chado_load_blast/load_blast/2.3.4+galaxy0",
             "tool_shed_repository": {
-                "changeset_revision": "f487ff676088",
-                "name": "tripal_analysis_sync",
+                "changeset_revision": "08ae8b27b193",
+                "name": "chado_load_blast",
                 "owner": "gga",
                 "tool_shed": "toolshed.g2.bx.psu.edu"
             },
-            "tool_state": "{\"analysis_id\": {\"__class__\": \"RuntimeValue\"}, \"wait_for\": {\"__class__\": \"ConnectedValue\"}, \"__page__\": null, \"__rerun_remap_job_id__\": null}",
-            "tool_version": "3.2.1.0",
+            "tool_state": "{\"analysis_id\": {\"__class__\": \"RuntimeValue\"}, \"blastdb_id\": \"21\", \"input\": {\"__class__\": \"RuntimeValue\"}, \"match_on_name\": \"false\", \"organism_id\": {\"__class__\": \"RuntimeValue\"}, \"psql_target\": {\"method\": \"remote\", \"__current_case__\": 0}, \"query_type\": \"polypeptide\", \"re_name\": \"\", \"skip_missing\": \"false\", \"wait_for\": {\"__class__\": \"RuntimeValue\"}, \"__page__\": null, \"__rerun_remap_job_id__\": null}",
+            "tool_version": "2.3.4+galaxy0",
             "type": "tool",
-            "uuid": "99e7496d-ac32-467d-8c09-2efd48d0231a",
+            "uuid": "a7ec5c91-7cef-4b9f-95a0-ed5542b8e142",
             "workflow_outputs": [
                 {
-                    "label": "Synchronize Analysis into Tripal",
+                    "label": null,
                     "output_name": "results",
-                    "uuid": "9792ea7a-0b4d-4355-9ce6-db2ba1ed36e9"
+                    "uuid": "119f219e-3d80-4b42-bb38-d07d4583048c"
                 }
             ]
         },
@@ -268,11 +280,20 @@
             "id": 5,
             "input_connections": {
                 "wait_for": {
-                    "id": 3,
+                    "id": 4,
                     "output_name": "results"
                 }
             },
-            "inputs": [],
+            "inputs": [
+                {
+                    "description": "runtime parameter for tool Synchronize an analysis",
+                    "name": "analysis_id"
+                },
+                {
+                    "description": "runtime parameter for tool Synchronize an analysis",
+                    "name": "wait_for"
+                }
+            ],
             "label": "sync blast analysis org2",
             "name": "Synchronize an analysis",
             "outputs": [
@@ -282,14 +303,14 @@
                 }
             ],
             "position": {
-                "bottom": 545.3999938964844,
-                "height": 154.39999389648438,
-                "left": 899,
-                "right": 1099,
-                "top": 391,
+                "bottom": 440.8000183105469,
+                "height": 153.60000610351562,
+                "left": 828.5,
+                "right": 1028.5,
+                "top": 287.20001220703125,
                 "width": 200,
-                "x": 899,
-                "y": 391
+                "x": 828.5,
+                "y": 287.20001220703125
             },
             "post_job_actions": {},
             "tool_id": "toolshed.g2.bx.psu.edu/repos/gga/tripal_analysis_sync/analysis_sync/3.2.1.0",
@@ -299,15 +320,15 @@
                 "owner": "gga",
                 "tool_shed": "toolshed.g2.bx.psu.edu"
             },
-            "tool_state": "{\"analysis_id\": \"1\", \"wait_for\": {\"__class__\": \"ConnectedValue\"}, \"__page__\": null, \"__rerun_remap_job_id__\": null}",
+            "tool_state": "{\"analysis_id\": {\"__class__\": \"RuntimeValue\"}, \"wait_for\": {\"__class__\": \"RuntimeValue\"}, \"__page__\": null, \"__rerun_remap_job_id__\": null}",
             "tool_version": "3.2.1.0",
             "type": "tool",
-            "uuid": "99f68436-37c7-409a-bcf0-70bde40a9a8e",
+            "uuid": "2fff7637-7904-46ff-87e1-ce2721727e75",
             "workflow_outputs": [
                 {
                     "label": "Synchronize Analysis into Tripal",
                     "output_name": "results",
-                    "uuid": "ce2c98b7-17f2-4646-a3bf-784af0103f4e"
+                    "uuid": "924991f3-6dd4-4752-9ce2-3832d72dff57"
                 }
             ]
         },
@@ -332,14 +353,14 @@
                 }
             ],
             "position": {
-                "bottom": 474.3999938964844,
-                "height": 154.39999389648438,
-                "left": 1174,
-                "right": 1374,
-                "top": 320,
+                "bottom": 368.8000030517578,
+                "height": 153.60000610351562,
+                "left": 1103.5,
+                "right": 1303.5,
+                "top": 215.1999969482422,
                 "width": 200,
-                "x": 1174,
-                "y": 320
+                "x": 1103.5,
+                "y": 215.1999969482422
             },
             "post_job_actions": {},
             "tool_id": "toolshed.g2.bx.psu.edu/repos/gga/tripal_db_populate_mviews/db_populate_mviews/3.2.1.0",
@@ -357,7 +378,7 @@
                 {
                     "label": "Populate Tripal materialized view(s)",
                     "output_name": "results",
-                    "uuid": "31970b1e-e567-4994-b05e-31d3406ebdf3"
+                    "uuid": "dc519305-8c27-4c53-9150-7dd37b5090cd"
                 }
             ]
         },
@@ -382,14 +403,14 @@
                 }
             ],
             "position": {
-                "bottom": 454.6000061035156,
-                "height": 113.60000610351562,
-                "left": 1444,
-                "right": 1644,
-                "top": 341,
+                "bottom": 349,
+                "height": 112.80000305175781,
+                "left": 1373.5,
+                "right": 1573.5,
+                "top": 236.1999969482422,
                 "width": 200,
-                "x": 1444,
-                "y": 341
+                "x": 1373.5,
+                "y": 236.1999969482422
             },
             "post_job_actions": {},
             "tool_id": "toolshed.g2.bx.psu.edu/repos/gga/tripal_db_index/db_index/3.2.1.1",
@@ -407,12 +428,12 @@
                 {
                     "label": "Index Tripal data",
                     "output_name": "results",
-                    "uuid": "c4398cf0-10ca-45c7-ab62-9854b7519ecb"
+                    "uuid": "e2911922-2412-4618-97fe-bcc783bb0865"
                 }
             ]
         }
     },
     "tags": [],
-    "uuid": "1219fcfa-e57c-437e-ac5e-4d803e3dafd9",
-    "version": 5
+    "uuid": "ffae97b5-698a-41a5-8561-470300594544",
+    "version": 6
 }
\ No newline at end of file