Release 2.0 (merge dev to master)

General

• Reorganize repository
• Fix verbosity mode
• Fix logs details
• Update README.md
• Update the "help" option in the scripts
• Update examples
• Factorize and optimize code
• Create file constants.py containing the constant variables

Input organisms.yml file

• Change root structure (a list instead of a dictionary, the name parameter is added to the species dictionary sp_dict)
• Add parameter main_species. Useful when there are multiple strains for the same species: set to yes to define the strain used as home for JBrowse
• Parameters species, genus, strain, sex, common will be sanitized (removing " ", "-", "(", ")", "'")
• Add jinja2 template

Input config file

• Change root structure (one-level dictionary)
• Add parameters authelia_secrets_env_path and authelia_db_postgres_password required for Authelia
• Change the name of some parameters (galaxy_persist_data, tripal_banner_path, jbrowse_menu_url)
• Remove unused apollo parameters (webapollo_user, webapollo_password)

Step "init"

• Update templates files
• Fix the copy of the specified banner
• Write galaxy_nginx.conf only if needed

Step "get data"

• Prefix data filenames with g_species_s
• Add the last modification date of the source file to the gff filename

Step "load data"

• Use Bioblend function upload_from_galaxy_fs() instead of upload_from_local() (to enable symlink, instead of copy)

Step "run workflow Phaeoexplorer"

• Development of a workflow loading fasta and gff files to Chado, Tripal and JBrowse, for one or two organisms of the same species

speciesData.py

@@ -4,10 +4,11 @@
 import os
 import sys
 import utilities
+import logging
+import constants
 
 from _datetime import datetime
 
-
 class SpeciesData:
     """
     This class contains attributes and functions to interact with the galaxy container of the GGA environment
@@ -15,61 +16,107 @@ class SpeciesData:
 
     """
 
+    def get_species_dir(self):
+
+        species_dir = None
+        if os.path.isdir(self.main_dir) and not self.genus_species is None:
+            species_dir = os.path.join(self.main_dir, self.genus_species)
+        else:
+            logging.error("Cannot set species dir with '{0}/{1}'".format(self.main_dir,self.genus_species))
+        return species_dir
+
+    def goto_species_dir(self):
+        """
+        Go to the species directory (starting from the main dir)
+
+        :return:
+        """
+
+        species_dir = self.get_species_dir()
+        try:
+            os.chdir(species_dir)
+        except OSError:
+            logging.critical("Cannot access %s" % species_dir)
+            sys.exit(0)
+        return 1
+
+    def clean_string(self, string):
+        if not string is None and string != "":
+            clean_string = string.replace(" ", "_").replace("-", "_").replace("(", "").replace(")", "").replace("'", "").strip()
+            return clean_string
+        else:
+            return string
+
     def __init__(self, parameters_dictionary):
-        # self.config_dictionary = None
         self.parameters_dictionary = parameters_dictionary
-        self.species = parameters_dictionary["description"]["species"].replace("(", "_").replace(")", "_").replace("-", "_")
-        self.genus = parameters_dictionary["description"]["genus"].replace("(", "_").replace(")", "_").replace("-", "_")
-        self.strain = parameters_dictionary["description"]["strain"].replace("(", "_").replace(")", "_").replace("-", "_")
-        self.sex = parameters_dictionary["description"]["sex"].replace("(", "_").replace(")", "_").replace("-", "_")
-        self.common = parameters_dictionary["description"]["common_name"].replace("(", "_").replace(")", "_").replace("-", "_")
+        self.name = parameters_dictionary[constants.ORG_PARAM_NAME]
+        parameters_dictionary_description=parameters_dictionary[constants.ORG_PARAM_DESC]
+        parameters_dictionary_data = parameters_dictionary[constants.ORG_PARAM_DATA]
+        parameters_dictionary_services = parameters_dictionary[constants.ORG_PARAM_SERVICES]
+
+        self.species = self.clean_string(parameters_dictionary_description[constants.ORG_PARAM_DESC_SPECIES])
+        self.genus = self.clean_string(parameters_dictionary_description[constants.ORG_PARAM_DESC_GENUS])
+        self.strain = self.clean_string(parameters_dictionary_description[constants.ORG_PARAM_DESC_STRAIN])
+        self.sex = self.clean_string(parameters_dictionary_description[constants.ORG_PARAM_DESC_SEX])
+        self.common = self.clean_string(parameters_dictionary_description[constants.ORG_PARAM_DESC_COMMON_NAME])
 
         self.date = datetime.today().strftime("%Y-%m-%d")
-        self.origin = parameters_dictionary["description"]["origin"]
-        self.performed = parameters_dictionary["data"]["performed_by"]
+        self.origin = parameters_dictionary_description[constants.ORG_PARAM_DESC_ORIGIN]
+        self.performed = parameters_dictionary_data[constants.ORG_PARAM_DATA_PERFORMED_BY]
 
-        if parameters_dictionary["data"]["genome_version"] == "":
+        if parameters_dictionary_data[constants.ORG_PARAM_DATA_GENOME_VERSION] == "":
             self.genome_version = "1.0"
         else:
-            self.genome_version = parameters_dictionary["data"]["genome_version"]
-        if parameters_dictionary["data"]["ogs_version"] == "":
+            self.genome_version = str(parameters_dictionary_data[constants.ORG_PARAM_DATA_GENOME_VERSION])
+        if parameters_dictionary_data[constants.ORG_PARAM_DATA_OGS_VERSION] == "":
             self.ogs_version = "1.0"
         else:
-            self.ogs_version = parameters_dictionary["data"]["ogs_version"]
+            self.ogs_version = str(parameters_dictionary_data[constants.ORG_PARAM_DATA_OGS_VERSION])
 
         # TODO: catch blocks if key is absent in input
-        self.genome_path = parameters_dictionary["data"]["genome_path"]
-        self.transcripts_path = parameters_dictionary["data"]["transcripts_path"]
-        self.proteins_path = parameters_dictionary["data"]["proteins_path"]
-        self.gff_path = parameters_dictionary["data"]["gff_path"]
-        self.interpro_path = parameters_dictionary["data"]["interpro_path"]
-        self.blastp_path = parameters_dictionary["data"]["blastp_path"]
-        self.blastx_path = parameters_dictionary["data"]["blastx_path"]
-        self.orthofinder_path = parameters_dictionary["data"]["orthofinder_path"]
+        self.genome_path = parameters_dictionary_data[constants.ORG_PARAM_DATA_GENOME_PATH]
+        self.transcripts_path = parameters_dictionary_data[constants.ORG_PARAM_DATA_TRANSCRIPTS_PATH]
+        self.proteins_path = parameters_dictionary_data[constants.ORG_PARAM_DATA_PROTEINS_PATH]
+        self.gff_path = parameters_dictionary_data[constants.ORG_PARAM_DATA_GFF_PATH]
+        self.interpro_path = parameters_dictionary_data[constants.ORG_PARAM_DATA_INTERPRO_PATH]
+        self.blastp_path = parameters_dictionary_data[constants.ORG_PARAM_DATA_BLASTP_PATH]
+        self.blastx_path = parameters_dictionary_data[constants.ORG_PARAM_DATA_BLASTX_PATH]
+        self.orthofinder_path = parameters_dictionary_data[constants.ORG_PARAM_DATA_ORTHOFINDER_PATH]
+
+        if(constants.ORG_PARAM_SERVICES_BLAST in parameters_dictionary_services.keys()):
+            self.blast = parameters_dictionary_services[constants.ORG_PARAM_SERVICES_BLAST]
+        else:
+            self.blast = "0"
 
         self.genus_lowercase = self.genus[0].lower() + self.genus[1:]
         self.genus_uppercase = self.genus[0].upper() + self.genus[1:]
         self.chado_species_name = "{0} {1}".format(self.species, self.sex)
         self.full_name = ' '.join(utilities.filter_empty_not_empty_items([self.genus_uppercase, self.species, self.strain, self.sex])["not_empty"])
-        self.full_name = self.full_name.replace("__", "_").replace("_ ", "_").replace(" _", "_")
-        if self.full_name.endswith("_") or self.full_name.endswith(" "):
-            self.full_name = self.full_name[0:-2]
 
         self.full_name_lowercase = self.full_name.lower()
         self.abbreviation = "_".join(utilities.filter_empty_not_empty_items([self.genus_lowercase[0], self.species, self.strain, self.sex])["not_empty"])
         
-        self.genus_species = self.genus_lowercase + "_" + self.species
+        self.genus_species = "{0}_{1}".format(self.genus.lower(), self.species.lower())
+        self.dataset_prefix = None
+        if self.sex is not None or self.sex != "":
+            self.dataset_prefix = self.genus[0].lower() + "_" + self.species.lower() + "_" + self.sex[0].lower()
+        else:
+            self.dataset_prefix = self.genus[0].lower() + "_" + self.species.lower()
+
+        # Bioblend/Chado IDs for an organism analyses/organisms/datasets/history/library
+        self.org_id = None
+        self.genome_analysis_id = None
+        self.ogs_analysis_id = None
         self.instance_url = None
         self.instance = None
         self.history_id = None
         self.library = None
         self.library_id = None
+
         self.script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
         self.main_dir = None
         self.species_dir = None
-        self.org_id = None
-        self.genome_analysis_id = None
-        self.ogs_analysis_id = None
+
         self.tool_panel = None
         self.datasets = dict()
         self.datasets_name = dict()
@@ -79,12 +126,6 @@ class SpeciesData:
         self.api_key = None  # API key used to communicate with the galaxy instance. Cannot be used to do user-tied actions
         self.datasets = dict()
         self.config = None  # Custom config used to set environment variables inside containers
-        self.species_folder_name = "_".join(utilities.filter_empty_not_empty_items([self.genus_lowercase.lower(), self.species.lower(), self.strain.lower(), self.sex.lower()])["not_empty"])
-        self.species_folder_name  = self.species_folder_name .replace("-", "_")
-        self.existing_folders_cache = {}
-        self.bam_metadata_cache = {}
-
-        # # Sanitize str attributes
-        # for var in vars(self):
-        #     for attr in var if type(attr) == str:
-        #         attr = attr.replace("(", "_").replace(")", "_")
+        self.species_folder_name = "_".join(utilities.filter_empty_not_empty_items(
+            [self.genus_lowercase.lower(), self.species.lower(), self.strain.lower(),
+             self.sex.lower()])["not_empty"])