Release 2.0 (merge dev to master)

General

• Reorganize repository
• Fix verbosity mode
• Fix logs details
• Update README.md
• Update the "help" option in the scripts
• Update examples
• Factorize and optimize code
• Create file constants.py containing the constant variables

Input organisms.yml file

• Change root structure (a list instead of a dictionary, the name parameter is added to the species dictionary sp_dict)
• Add parameter main_species. Useful when there are multiple strains for the same species: set to yes to define the strain used as home for JBrowse
• Parameters species, genus, strain, sex, common will be sanitized (removing " ", "-", "(", ")", "'")
• Add jinja2 template

Input config file

• Change root structure (one-level dictionary)
• Add parameters authelia_secrets_env_path and authelia_db_postgres_password required for Authelia
• Change the name of some parameters (galaxy_persist_data, tripal_banner_path, jbrowse_menu_url)
• Remove unused apollo parameters (webapollo_user, webapollo_password)

Step "init"

• Update templates files
• Fix the copy of the specified banner
• Write galaxy_nginx.conf only if needed

Step "get data"

• Prefix data filenames with g_species_s
• Add the last modification date of the source file to the gff filename

Step "load data"

• Use Bioblend function upload_from_galaxy_fs() instead of upload_from_local() (to enable symlink, instead of copy)

Step "run workflow Phaeoexplorer"

• Development of a workflow loading fasta and gff files to Chado, Tripal and JBrowse, for one or two organisms of the same species

workflows_phaeoexplorer/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_1org_v4.ga

+{
+    "a_galaxy_workflow": "true",
+    "annotation": "",
+    "format-version": "0.1",
+    "name": "chado_load_tripal_synchronize_jbrowse_1org_v4",
+    "steps": {
+        "0": {
+            "annotation": "",
+            "content_id": null,
+            "errors": null,
+            "id": 0,
+            "input_connections": {},
+            "inputs": [
+                {
+                    "description": "",
+                    "name": "genome"
+                }
+            ],
+            "label": "genome",
+            "name": "Input dataset",
+            "outputs": [],
+            "position": {
+                "bottom": 277.1999969482422,
+                "height": 61.19999694824219,
+                "left": 436.5,
+                "right": 636.5,
+                "top": 216,
+                "width": 200,
+                "x": 436.5,
+                "y": 216
+            },
+            "tool_id": null,
+            "tool_state": "{\"optional\": false}",
+            "tool_version": null,
+            "type": "data_input",
+            "uuid": "fa9981ea-4012-40aa-ad84-6e6f61049104",
+            "workflow_outputs": [
+                {
+                    "label": null,
+                    "output_name": "output",
+                    "uuid": "65bba69d-b8f0-4f7e-a66a-71afa9a8975f"
+                }
+            ]
+        },
+        "1": {
+            "annotation": "",
+            "content_id": null,
+            "errors": null,
+            "id": 1,
+            "input_connections": {},
+            "inputs": [
+                {
+                    "description": "",
+                    "name": "annotations"
+                }
+            ],
+            "label": "annotations",
+            "name": "Input dataset",
+            "outputs": [],
+            "position": {
+                "bottom": 367.1999969482422,
+                "height": 61.19999694824219,
+                "left": 467.5,
+                "right": 667.5,
+                "top": 306,
+                "width": 200,
+                "x": 467.5,
+                "y": 306
+            },
+            "tool_id": null,
+            "tool_state": "{\"optional\": false}",
+            "tool_version": null,
+            "type": "data_input",
+            "uuid": "61d23082-b459-4014-8584-6ff5b98ce689",
+            "workflow_outputs": [
+                {
+                    "label": null,
+                    "output_name": "output",
+                    "uuid": "acb766e7-fcd7-42e2-8fcf-638024338fc4"
+                }
+            ]
+        },
+        "2": {
+            "annotation": "",
+            "content_id": null,
+            "errors": null,
+            "id": 2,
+            "input_connections": {},
+            "inputs": [
+                {
+                    "description": "",
+                    "name": "proteins"
+                }
+            ],
+            "label": "proteins",
+            "name": "Input dataset",
+            "outputs": [],
+            "position": {
+                "bottom": 456.1999969482422,
+                "height": 61.19999694824219,
+                "left": 489.5,
+                "right": 689.5,
+                "top": 395,
+                "width": 200,
+                "x": 489.5,
+                "y": 395
+            },
+            "tool_id": null,
+            "tool_state": "{\"optional\": false}",
+            "tool_version": null,
+            "type": "data_input",
+            "uuid": "ea25f583-f55d-4fdd-a7a9-86ffb4b9c731",
+            "workflow_outputs": [
+                {
+                    "label": null,
+                    "output_name": "output",
+                    "uuid": "eba4266d-d468-448f-8ce3-fa87b497cbf8"
+                }
+            ]
+        },
+        "3": {
+            "annotation": "",
+            "content_id": "toolshed.g2.bx.psu.edu/repos/gga/chado_feature_load_fasta/feature_load_fasta/2.3.4+galaxy0",
+            "errors": null,
+            "id": 3,
+            "input_connections": {
+                "fasta": {
+                    "id": 0,
+                    "output_name": "output"
+                }
+            },
+            "inputs": [
+                {
+                    "description": "runtime parameter for tool Chado load fasta",
+                    "name": "analysis_id"
+                },
+                {
+                    "description": "runtime parameter for tool Chado load fasta",
+                    "name": "fasta"
+                },
+                {
+                    "description": "runtime parameter for tool Chado load fasta",
+                    "name": "organism"
+                },
+                {
+                    "description": "runtime parameter for tool Chado load fasta",
+                    "name": "wait_for"
+                }
+            ],
+            "label": null,
+            "name": "Chado load fasta",
+            "outputs": [
+                {
+                    "name": "results",
+                    "type": "json"
+                }
+            ],
+            "position": {
+                "bottom": 356.1999969482422,
+                "height": 143.1999969482422,
+                "left": 766.5,
+                "right": 966.5,
+                "top": 213,
+                "width": 200,
+                "x": 766.5,
+                "y": 213
+            },
+            "post_job_actions": {},
+            "tool_id": "toolshed.g2.bx.psu.edu/repos/gga/chado_feature_load_fasta/feature_load_fasta/2.3.4+galaxy0",
+            "tool_shed_repository": {
+                "changeset_revision": "ba4d07fbaf47",
+                "name": "chado_feature_load_fasta",
+                "owner": "gga",
+                "tool_shed": "toolshed.g2.bx.psu.edu"
+            },
+            "tool_state": "{\"analysis_id\": {\"__class__\": \"RuntimeValue\"}, \"do_update\": \"false\", \"ext_db\": {\"db\": \"\", \"re_db_accession\": \"\"}, \"fasta\": {\"__class__\": \"RuntimeValue\"}, \"match_on_name\": \"false\", \"organism\": {\"__class__\": \"RuntimeValue\"}, \"psql_target\": {\"method\": \"remote\", \"__current_case__\": 0}, \"re_name\": \"\", \"re_uniquename\": \"\", \"relationships\": {\"rel_type\": \"none\", \"__current_case__\": 0}, \"sequence_type\": \"contig\", \"wait_for\": {\"__class__\": \"RuntimeValue\"}, \"__page__\": null, \"__rerun_remap_job_id__\": null}",
+            "tool_version": "2.3.4+galaxy0",
+            "type": "tool",
+            "uuid": "86b4962b-d001-44f3-b2f5-349e0daccc69",
+            "workflow_outputs": [
+                {
+                    "label": null,
+                    "output_name": "results",
+                    "uuid": "892534b5-0d67-44da-8892-f17da2be9e9c"
+                }
+            ]
+        },
+        "4": {
+            "annotation": "",
+            "content_id": "toolshed.g2.bx.psu.edu/repos/iuc/jbrowse/jbrowse/1.16.11+galaxy0",
+            "errors": null,
+            "id": 4,
+            "input_connections": {
+                "reference_genome|genome": {
+                    "id": 0,
+                    "output_name": "output"
+                },
+                "track_groups_0|data_tracks_0|data_format|annotation": {
+                    "id": 1,
+                    "output_name": "output"
+                }
+            },
+            "inputs": [
+                {
+                    "description": "runtime parameter for tool JBrowse",
+                    "name": "reference_genome"
+                }
+            ],
+            "label": null,
+            "name": "JBrowse",
+            "outputs": [
+                {
+                    "name": "output",
+                    "type": "html"
+                }
+            ],
+            "position": {
+                "bottom": 572,
+                "height": 184,
+                "left": 753.5,
+                "right": 953.5,
+                "top": 388,
+                "width": 200,
+                "x": 753.5,
+                "y": 388
+            },
+            "post_job_actions": {},
+            "tool_id": "toolshed.g2.bx.psu.edu/repos/iuc/jbrowse/jbrowse/1.16.11+galaxy0",
+            "tool_shed_repository": {
+                "changeset_revision": "4542035c1075",
+                "name": "jbrowse",
+                "owner": "iuc",
+                "tool_shed": "toolshed.g2.bx.psu.edu"
+            },
+            "tool_state": "{\"action\": {\"action_select\": \"create\", \"__current_case__\": 0}, \"gencode\": \"1\", \"jbgen\": {\"defaultLocation\": \"\", \"trackPadding\": \"20\", \"shareLink\": \"true\", \"aboutDescription\": \"\", \"show_tracklist\": \"true\", \"show_nav\": \"true\", \"show_overview\": \"true\", \"show_menu\": \"true\", \"hideGenomeOptions\": \"false\"}, \"plugins\": {\"BlastView\": \"true\", \"ComboTrackSelector\": \"false\", \"GCContent\": \"false\"}, \"reference_genome\": {\"genome_type_select\": \"history\", \"__current_case__\": 1, \"genome\": {\"__class__\": \"RuntimeValue\"}}, \"standalone\": \"minimal\", \"track_groups\": [{\"__index__\": 0, \"category\": \"Annotation\", \"data_tracks\": [{\"__index__\": 0, \"data_format\": {\"data_format_select\": \"gene_calls\", \"__current_case__\": 2, \"annotation\": {\"__class__\": \"RuntimeValue\"}, \"match_part\": {\"match_part_select\": \"false\", \"__current_case__\": 1}, \"index\": \"false\", \"track_config\": {\"track_class\": \"NeatHTMLFeatures/View/Track/NeatFeatures\", \"__current_case__\": 3, \"html_options\": {\"topLevelFeatures\": \"\"}}, \"jbstyle\": {\"style_classname\": \"transcript\", \"style_label\": \"product,name,id\", \"style_description\": \"note,description\", \"style_height\": \"10px\", \"max_height\": \"600\"}, \"jbcolor_scale\": {\"color_score\": {\"color_score_select\": \"none\", \"__current_case__\": 0, \"color\": {\"color_select\": \"automatic\", \"__current_case__\": 0}}}, \"jb_custom_config\": {\"option\": []}, \"jbmenu\": {\"track_menu\": [{\"__index__\": 0, \"menu_action\": \"iframeDialog\", \"menu_label\": \"View transcript report\", \"menu_title\": \"Transcript {id}\", \"menu_url\": \"__MENU_URL_ORG__\", \"menu_icon\": \"dijitIconBookmark\"}]}, \"track_visibility\": \"default_off\", \"override_apollo_plugins\": \"False\", \"override_apollo_drag\": \"False\"}}]}], \"uglyTestingHack\": \"\", \"__page__\": null, \"__rerun_remap_job_id__\": null}",
+            "tool_version": "1.16.11+galaxy0",
+            "type": "tool",
+            "uuid": "4e87e6b5-c37c-4429-a491-2d6a411d8a13",
+            "workflow_outputs": [
+                {
+                    "label": null,
+                    "output_name": "output",
+                    "uuid": "37d09194-4527-4428-963c-85cb351efcba"
+                }
+            ]
+        },
+        "5": {
+            "annotation": "",
+            "content_id": "toolshed.g2.bx.psu.edu/repos/gga/chado_feature_load_gff/feature_load_gff/2.3.4+galaxy0",
+            "errors": null,
+            "id": 5,
+            "input_connections": {
+                "fasta": {
+                    "id": 2,
+                    "output_name": "output"
+                },
+                "gff": {
+                    "id": 1,
+                    "output_name": "output"
+                },
+                "wait_for": {
+                    "id": 3,
+                    "output_name": "results"
+                }
+            },
+            "inputs": [
+                {
+                    "description": "runtime parameter for tool Chado load gff",
+                    "name": "analysis_id"
+                },
+                {
+                    "description": "runtime parameter for tool Chado load gff",
+                    "name": "fasta"
+                },
+                {
+                    "description": "runtime parameter for tool Chado load gff",
+                    "name": "gff"
+                },
+                {
+                    "description": "runtime parameter for tool Chado load gff",
+                    "name": "organism"
+                },
+                {
+                    "description": "runtime parameter for tool Chado load gff",
+                    "name": "wait_for"
+                }
+            ],
+            "label": null,
+            "name": "Chado load gff",
+            "outputs": [
+                {
+                    "name": "results",
+                    "type": "txt"
+                }
+            ],
+            "position": {
+                "bottom": 383.6000061035156,
+                "height": 173.60000610351562,
+                "left": 1043.5,
+                "right": 1243.5,
+                "top": 210,
+                "width": 200,
+                "x": 1043.5,
+                "y": 210
+            },
+            "post_job_actions": {},
+            "tool_id": "toolshed.g2.bx.psu.edu/repos/gga/chado_feature_load_gff/feature_load_gff/2.3.4+galaxy0",
+            "tool_shed_repository": {
+                "changeset_revision": "e9a6d7568817",
+                "name": "chado_feature_load_gff",
+                "owner": "gga",
+                "tool_shed": "toolshed.g2.bx.psu.edu"
+            },
+            "tool_state": "{\"add_only\": \"false\", \"analysis_id\": {\"__class__\": \"RuntimeValue\"}, \"fasta\": {\"__class__\": \"RuntimeValue\"}, \"gff\": {\"__class__\": \"RuntimeValue\"}, \"landmark_type\": \"contig\", \"no_seq_compute\": \"false\", \"organism\": {\"__class__\": \"RuntimeValue\"}, \"prot_naming\": {\"method\": \"regex\", \"__current_case__\": 1, \"re_protein_capture\": \"^mRNA(_.+)$\", \"re_protein\": \"prot\\\\1\"}, \"psql_target\": {\"method\": \"remote\", \"__current_case__\": 0}, \"wait_for\": {\"__class__\": \"RuntimeValue\"}, \"__page__\": null, \"__rerun_remap_job_id__\": null}",
+            "tool_version": "2.3.4+galaxy0",
+            "type": "tool",
+            "uuid": "34eb5e35-988e-4b49-8f2c-b11761a43588",
+            "workflow_outputs": [
+                {
+                    "label": null,
+                    "output_name": "results",
+                    "uuid": "2c0c2e3d-7ce1-4f7e-ad2f-456d9e97fdb8"
+                }
+            ]
+        },
+        "6": {
+            "annotation": "",
+            "content_id": "toolshed.g2.bx.psu.edu/repos/gga/jbrowse_to_container/jbrowse_to_container/0.5.1",
+            "errors": null,
+            "id": 6,
+            "input_connections": {
+                "organisms_0|jbrowse": {
+                    "id": 4,
+                    "output_name": "output"
+                }
+            },
+            "inputs": [],
+            "label": null,
+            "name": "Add organisms to JBrowse container",
+            "outputs": [
+                {
+                    "name": "output",
+                    "type": "html"
+                }
+            ],
+            "position": {
+                "bottom": 551.1999969482422,
+                "height": 133.1999969482422,
+                "left": 1039.5,
+                "right": 1239.5,
+                "top": 418,
+                "width": 200,
+                "x": 1039.5,
+                "y": 418
+            },
+            "post_job_actions": {},
+            "tool_id": "toolshed.g2.bx.psu.edu/repos/gga/jbrowse_to_container/jbrowse_to_container/0.5.1",
+            "tool_shed_repository": {
+                "changeset_revision": "11033bdad2ca",
+                "name": "jbrowse_to_container",
+                "owner": "gga",
+                "tool_shed": "toolshed.g2.bx.psu.edu"
+            },
+            "tool_state": "{\"organisms\": [{\"__index__\": 0, \"jbrowse\": {\"__class__\": \"RuntimeValue\"}, \"name\": \"__DISPLAY_NAME_ORG__\", \"advanced\": {\"unique_id\": \"__UNIQUE_ID_ORG__\"}}], \"__page__\": null, \"__rerun_remap_job_id__\": null}",
+            "tool_version": "0.5.1",
+            "type": "tool",
+            "uuid": "f06c3ec7-936f-41be-9718-248b1d760d11",
+            "workflow_outputs": [
+                {
+                    "label": null,
+                    "output_name": "output",
+                    "uuid": "948e9774-03e5-43be-ad57-4b785372f78a"
+                }
+            ]
+        },
+        "7": {
+            "annotation": "",
+            "content_id": "toolshed.g2.bx.psu.edu/repos/gga/tripal_feature_sync/feature_sync/3.2.1.0",
+            "errors": null,
+            "id": 7,
+            "input_connections": {
+                "wait_for": {
+                    "id": 5,
+                    "output_name": "results"
+                }
+            },
+            "inputs": [
+                {
+                    "description": "runtime parameter for tool Synchronize features",
+                    "name": "organism_id"
+                }
+            ],
+            "label": null,
+            "name": "Synchronize features",
+            "outputs": [
+                {
+                    "name": "results",
+                    "type": "txt"
+                }
+            ],
+            "position": {
+                "bottom": 370.6000061035156,
+                "height": 153.60000610351562,
+                "left": 1325.5,
+                "right": 1525.5,
+                "top": 217,
+                "width": 200,
+                "x": 1325.5,
+                "y": 217
+            },
+            "post_job_actions": {},
+            "tool_id": "toolshed.g2.bx.psu.edu/repos/gga/tripal_feature_sync/feature_sync/3.2.1.0",
+            "tool_shed_repository": {
+                "changeset_revision": "64e36c3f0dd6",
+                "name": "tripal_feature_sync",
+                "owner": "gga",
+                "tool_shed": "toolshed.g2.bx.psu.edu"
+            },
+            "tool_state": "{\"organism_id\": {\"__class__\": \"RuntimeValue\"}, \"repeat_ids\": [], \"repeat_types\": [{\"__index__\": 0, \"types\": \"mRNA\"}, {\"__index__\": 1, \"types\": \"polypeptide\"}], \"wait_for\": {\"__class__\": \"ConnectedValue\"}, \"__page__\": null, \"__rerun_remap_job_id__\": null}",
+            "tool_version": "3.2.1.0",
+            "type": "tool",
+            "uuid": "38dd9a7d-46e1-48c6-8a5a-9c62a6860431",
+            "workflow_outputs": [
+                {
+                    "label": null,
+                    "output_name": "results",
+                    "uuid": "704c9382-8a3e-4b2b-8190-399a43e6455f"
+                }
+            ]
+        },
+        "8": {
+            "annotation": "",
+            "content_id": "toolshed.g2.bx.psu.edu/repos/gga/tripal_db_populate_mviews/db_populate_mviews/3.2.1.0",
+            "errors": null,
+            "id": 8,
+            "input_connections": {
+                "wait_for": {
+                    "id": 7,
+                    "output_name": "results"
+                }
+            },
+            "inputs": [],
+            "label": null,
+            "name": "Populate materialized views",
+            "outputs": [
+                {
+                    "name": "results",
+                    "type": "txt"
+                }
+            ],
+            "position": {
+                "bottom": 369.6000061035156,
+                "height": 153.60000610351562,
+                "left": 1611.5,
+                "right": 1811.5,
+                "top": 216,
+                "width": 200,
+                "x": 1611.5,
+                "y": 216
+            },
+            "post_job_actions": {},
+            "tool_id": "toolshed.g2.bx.psu.edu/repos/gga/tripal_db_populate_mviews/db_populate_mviews/3.2.1.0",
+            "tool_shed_repository": {
+                "changeset_revision": "3c08f32a3dc1",
+                "name": "tripal_db_populate_mviews",
+                "owner": "gga",
+                "tool_shed": "toolshed.g2.bx.psu.edu"
+            },
+            "tool_state": "{\"mview\": \"\", \"wait_for\": {\"__class__\": \"ConnectedValue\"}, \"__page__\": null, \"__rerun_remap_job_id__\": null}",
+            "tool_version": "3.2.1.0",
+            "type": "tool",
+            "uuid": "b34ddfee-c317-4c21-99a6-679bd640b1be",
+            "workflow_outputs": [
+                {
+                    "label": null,
+                    "output_name": "results",
+                    "uuid": "53919587-2586-4c0f-9ae6-082119727c97"
+                }
+            ]
+        },
+        "9": {
+            "annotation": "",
+            "content_id": "toolshed.g2.bx.psu.edu/repos/gga/tripal_db_index/db_index/3.2.1.1",
+            "errors": null,
+            "id": 9,
+            "input_connections": {
+                "wait_for": {
+                    "id": 8,
+                    "output_name": "results"
+                }
+            },
+            "inputs": [],
+            "label": null,
+            "name": "Index Tripal data",
+            "outputs": [
+                {
+                    "name": "results",
+                    "type": "txt"
+                }
+            ],
+            "position": {
+                "bottom": 328.8000030517578,
+                "height": 112.80000305175781,
+                "left": 1897.5,
+                "right": 2097.5,
+                "top": 216,
+                "width": 200,
+                "x": 1897.5,
+                "y": 216
+            },
+            "post_job_actions": {},
+            "tool_id": "toolshed.g2.bx.psu.edu/repos/gga/tripal_db_index/db_index/3.2.1.1",
+            "tool_shed_repository": {
+                "changeset_revision": "d55a39f12dda",
+                "name": "tripal_db_index",
+                "owner": "gga",
+                "tool_shed": "toolshed.g2.bx.psu.edu"
+            },
+            "tool_state": "{\"expose\": {\"do_expose\": \"no\", \"__current_case__\": 0}, \"queues\": \"10\", \"table\": {\"mode\": \"website\", \"__current_case__\": 0}, \"tokenizer\": \"standard\", \"wait_for\": {\"__class__\": \"ConnectedValue\"}, \"__page__\": null, \"__rerun_remap_job_id__\": null}",
+            "tool_version": "3.2.1.1",
+            "type": "tool",
+            "uuid": "6f8cf6b5-82f2-40bf-80c0-aecf74bedd5a",
+            "workflow_outputs": [
+                {
+                    "label": null,
+                    "output_name": "results",
+                    "uuid": "65a75a90-c47f-4cb4-be74-87cd89d1988a"
+                }
+            ]
+        }
+    },
+    "tags": [],
+    "uuid": "f8c6fa33-4ade-4251-a214-0ce77cdaac6e",
+    "version": 1
+}
\ No newline at end of file