Release 2.0 (merge dev to master)

General

• Reorganize repository
• Fix verbosity mode
• Fix logs details
• Update README.md
• Update the "help" option in the scripts
• Update examples
• Factorize and optimize code
• Create file constants.py containing the constant variables

Input organisms.yml file

• Change root structure (a list instead of a dictionary, the name parameter is added to the species dictionary sp_dict)
• Add parameter main_species. Useful when there are multiple strains for the same species: set to yes to define the strain used as home for JBrowse
• Parameters species, genus, strain, sex, common will be sanitized (removing " ", "-", "(", ")", "'")
• Add jinja2 template

Input config file

• Change root structure (one-level dictionary)
• Add parameters authelia_secrets_env_path and authelia_db_postgres_password required for Authelia
• Change the name of some parameters (galaxy_persist_data, tripal_banner_path, jbrowse_menu_url)
• Remove unused apollo parameters (webapollo_user, webapollo_password)

Step "init"

• Update templates files
• Fix the copy of the specified banner
• Write galaxy_nginx.conf only if needed

Step "get data"

• Prefix data filenames with g_species_s
• Add the last modification date of the source file to the gff filename

Step "load data"

• Use Bioblend function upload_from_galaxy_fs() instead of upload_from_local() (to enable symlink, instead of copy)

Step "run workflow Phaeoexplorer"

• Development of a workflow loading fasta and gff files to Chado, Tripal and JBrowse, for one or two organisms of the same species

utilities.py

@@ -6,7 +6,26 @@ import logging
 import sys
 import os
 import subprocess
+import bioblend
+import constants
 
+def load_yaml(yaml_file):
+
+    try:
+        with open(yaml_file, 'r') as stream:
+            try:
+                data = yaml.safe_load(stream)
+            except yaml.YAMLError as err:
+                logging.critical("Input file %s is not in YAML format" % yaml_file)
+                sys.exit(err)
+    except FileNotFoundError:
+        logging.critical("Input file doesn't exist (%s)" % yaml_file)
+        sys.exit()
+    except OSError:
+        logging.critical("Input file cannot be read (%s)" % yaml_file)
+        sys.exit()
+
+    return data
 
 def parse_config(config_file):
     """
@@ -16,25 +35,14 @@ def parse_config(config_file):
     :return:
     """
 
-    config_variables = {}
-    logging.debug("Using config: %s" % os.path.abspath(config_file))
-    try:
-        with open(config_file, 'r') as stream:
-            yaml_dict = yaml.safe_load(stream)
-            for k, v in yaml_dict.items():
-                for k2, v2 in v.items():
-                    config_variables[k2] = v2  # Add a key:value pair to variables for replacement in the compose template file
-
-    except FileNotFoundError:
-        logging.critical("The config file specified doesn't exist (%s)" % config_file)
-        sys.exit()
-    except OSError:
-        logging.critical("The config file specified cannot be read (%s)" % config_file)
+    config_dict = load_yaml(config_file)
+    if isinstance(config_dict, dict):
+        #logging.debug("Config dictionary: {0}".format(config_dict))
+        return config_dict
+    else:
+        logging.critical("Config yaml file is not a dictionary" % config_file)
         sys.exit()
 
-    return config_variables
-
-
 def parse_input(input_file):
     """
     Parse the yml input file to extract data to create the SpeciesData objects
@@ -44,27 +52,13 @@ def parse_input(input_file):
     :return:
     """
 
-    parsed_sp_dict_list = []
-
-    try:
-        with open(input_file, 'r') as stream:
-            try:
-                yaml_dict = yaml.safe_load(stream)
-                for k, v in yaml_dict.items():
-                    parsed_sp_dict_list.append(v)
-            except yaml.YAMLError as err:
-                logging.critical("Input file is not in YAML format")
-                sys.exit(err)
-    except FileNotFoundError:
-        logging.critical("The specified input file doesn't exist (%s)" % input_file)
-        sys.exit()
-    except OSError:
-        logging.critical("The specified input file cannot be read (%s)" % input_file)
+    sp_dict_list = load_yaml(input_file)
+    if isinstance(sp_dict_list, list):
+        return sp_dict_list
+    else:
+        logging.critical("Input organisms yaml file is not a list" % input_file)
         sys.exit()
 
-    return parsed_sp_dict_list
-
-
 def filter_empty_not_empty_items(li):
     """
     Separate a list between empty items and non empty items.
@@ -123,6 +117,12 @@ def get_species_history_id(instance, full_name):
 
     return [history_id, show_history]
 
+def get_gspecies_string_from_sp_dict(sp_dict):
+
+    genus = sp_dict[constants.ORG_PARAM_DESC][constants.ORG_PARAM_DESC_GENUS]
+    species = sp_dict[constants.ORG_PARAM_DESC][constants.ORG_PARAM_DESC_SPECIES]
+    gspecies = genus.lower() + "_" + species.lower()
+    return gspecies
 
 def get_unique_species_str_list(sp_dict_list):
     """
@@ -137,15 +137,9 @@ def get_unique_species_str_list(sp_dict_list):
     unique_species_li = []
 
     for sp in sp_dict_list:
-        for k, v in sp.items():
-            sp_gspecies = ""
-            for k2, v2 in v.items():
-                if k2 == "genus":
-                    sp_gspecies = sp_gspecies.lower() + v2
-                elif k2 == "species":
-                    sp_gspecies = sp_gspecies.lower() + "_" + v2
-            if sp_gspecies not in unique_species_li and sp_gspecies != "":
-                unique_species_li.append(sp_gspecies)
+        sp_gspecies = get_gspecies_string_from_sp_dict(sp)
+        if sp_gspecies not in unique_species_li and sp_gspecies != "":
+            unique_species_li.append(sp_gspecies)
                 
     return unique_species_li
 
@@ -162,16 +156,70 @@ def get_unique_species_dict_list(sp_dict_list):
 
     unique_species_dict = {}
     unique_species_list_of_dict = []
-    unique_species_genus_species = get_unique_species_str_list(sp_dict_list=sp_dict_list)
 
     for sp in sp_dict_list:
-        for gspecies in unique_species_genus_species:
-            if gspecies not in unique_species_dict.keys():
-                unique_species_dict[gspecies] = sp
-            else:
-                continue
+        gspecies = get_gspecies_string_from_sp_dict(sp)
+        if gspecies not in unique_species_dict.keys() or ( constants.ORG_PARAM_DESC_MAIN_SPECIES in sp[constants.ORG_PARAM_DESC].keys() and
+                                                           sp[constants.ORG_PARAM_DESC][constants.ORG_PARAM_DESC_MAIN_SPECIES] == True ) :
+            unique_species_dict[gspecies] = sp
+        else:
+            continue
 
     for k, v in unique_species_dict.items():
         unique_species_list_of_dict.append(v)
 
     return unique_species_list_of_dict
+
+def run_tool(instance, tool_id, history_id, tool_inputs):
+
+    output_dict = None
+    try:
+        logging.debug("Running tool {0} with tool inputs: {1}".format(tool_id, tool_inputs))
+        output_dict =  instance.tools.run_tool(
+            tool_id=tool_id,
+            history_id=history_id,
+            tool_inputs=tool_inputs)
+    except bioblend.ConnectionError:
+        logging.error("Unexpected HTTP response (bioblend.ConnectionError) when running tool {0} with tool inputs: {1}".format(tool_id, tool_inputs))
+
+    return output_dict
+
+def run_tool_and_get_single_output_dataset_id(instance, tool_id, history_id, tool_inputs):
+
+    output_dict = run_tool(instance, tool_id, history_id, tool_inputs)
+    single_output_dataset_id = output_dict["outputs"][0]["id"]
+
+    return single_output_dataset_id
+
+def create_org_param_dict_from_constants():
+    """
+    Create a dictionary of variables containing the keys needed to render the organisms.yml.j2 (NOT the values)
+    Created from the constants
+    """
+
+    org_param_dict={}
+    org_param_dict["org_param_name"] = constants.ORG_PARAM_NAME
+    org_param_dict["org_param_desc"] = constants.ORG_PARAM_DESC
+    org_param_dict["org_param_desc_genus"] = constants.ORG_PARAM_DESC_GENUS
+    org_param_dict["org_param_desc_species"] = constants.ORG_PARAM_DESC_SPECIES
+    org_param_dict["org_param_desc_sex"] = constants.ORG_PARAM_DESC_SEX
+    org_param_dict["org_param_desc_strain"] = constants.ORG_PARAM_DESC_STRAIN
+    org_param_dict["org_param_desc_common_name"] = constants.ORG_PARAM_DESC_COMMON_NAME
+    org_param_dict["org_param_desc_origin"] = constants.ORG_PARAM_DESC_ORIGIN
+    org_param_dict["org_param_desc_main_species"] = constants.ORG_PARAM_DESC_MAIN_SPECIES
+    org_param_dict["org_param_data"] = constants.ORG_PARAM_DATA
+    org_param_dict["org_param_data_genome_path"] = constants.ORG_PARAM_DATA_GENOME_PATH
+    org_param_dict["org_param_data_transcripts_path"] = constants.ORG_PARAM_DATA_TRANSCRIPTS_PATH
+    org_param_dict["org_param_data_proteins_path"] = constants.ORG_PARAM_DATA_PROTEINS_PATH
+    org_param_dict["org_param_data_gff_path"] = constants.ORG_PARAM_DATA_GFF_PATH
+    org_param_dict["org_param_data_interpro_path"] = constants.ORG_PARAM_DATA_INTERPRO_PATH
+    org_param_dict["org_param_data_orthofinder_path"] = constants.ORG_PARAM_DATA_ORTHOFINDER_PATH
+    org_param_dict["org_param_data_blastp_path"] = constants.ORG_PARAM_DATA_BLASTP_PATH
+    org_param_dict["org_param_data_blastx_path"] = constants.ORG_PARAM_DATA_BLASTX_PATH
+    org_param_dict["org_param_data_genome_version"] = constants.ORG_PARAM_DATA_GENOME_VERSION
+    org_param_dict["org_param_data_ogs_version"] = constants.ORG_PARAM_DATA_OGS_VERSION
+    org_param_dict["org_param_data_performed_by"] = constants.ORG_PARAM_DATA_PERFORMED_BY
+    org_param_dict["org_param_services"] = constants.ORG_PARAM_SERVICES
+    org_param_dict["org_param_services_blast"] = constants.ORG_PARAM_SERVICES_BLAST
+
+    return org_param_dict
\ No newline at end of file