Release 2.0 (merge dev to master)

General

• Reorganize repository
• Fix verbosity mode
• Fix logs details
• Update README.md
• Update the "help" option in the scripts
• Update examples
• Factorize and optimize code
• Create file constants.py containing the constant variables

Input organisms.yml file

• Change root structure (a list instead of a dictionary, the name parameter is added to the species dictionary sp_dict)
• Add parameter main_species. Useful when there are multiple strains for the same species: set to yes to define the strain used as home for JBrowse
• Parameters species, genus, strain, sex, common will be sanitized (removing " ", "-", "(", ")", "'")
• Add jinja2 template

Input config file

• Change root structure (one-level dictionary)
• Add parameters authelia_secrets_env_path and authelia_db_postgres_password required for Authelia
• Change the name of some parameters (galaxy_persist_data, tripal_banner_path, jbrowse_menu_url)
• Remove unused apollo parameters (webapollo_user, webapollo_password)

Step "init"

• Update templates files
• Fix the copy of the specified banner
• Write galaxy_nginx.conf only if needed

Step "get data"

• Prefix data filenames with g_species_s
• Add the last modification date of the source file to the gff filename

Step "load data"

• Use Bioblend function upload_from_galaxy_fs() instead of upload_from_local() (to enable symlink, instead of copy)

Step "run workflow Phaeoexplorer"

• Development of a workflow loading fasta and gff files to Chado, Tripal and JBrowse, for one or two organisms of the same species

gga_load_data.py

 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
+import re
 
 import bioblend
 import argparse
 import os
-import subprocess
 import logging
 import sys
-import fnmatch
 import time
 import json
-import re
-import stat
-import shutil
-
-from bioblend.galaxy.objects import GalaxyInstance
+import yaml
+import subprocess
 from bioblend import galaxy
+from bioblend.galaxy.objects import GalaxyInstance
 
 import utilities
 import speciesData
+import constants
 
 """ 
 gga_load_data.py
 
-Usage: $ python3 gga_init.py -i input_example.yml --config config.yml [OPTIONS]
+Usage: $ python3 gga_load_data.py -i input_example.yml --config config.yml [OPTIONS]
 
 Do not call this script before the galaxy container is ready
 """
 
-
 class LoadData(speciesData.SpeciesData):
     """
     Child of SpeciesData
@@ -38,41 +35,29 @@ class LoadData(speciesData.SpeciesData):
     Optional data file formatting
 
     """
+    def __init__(self, parameters_dictionary):
+        self.existing_folders_cache = {}
+        self.bam_metadata_cache = {}
+        super().__init__(parameters_dictionary)
 
-    def goto_species_dir(self):
-        """
-        Go to the species directory (starting from the main dir)
-
-        :return:
-        """
-
-        os.chdir(self.main_dir)
-        species_dir = os.path.join(self.main_dir, self.genus_species) + "/"
-        try:
-            os.chdir(species_dir)
-        except OSError:
-            logging.critical("Cannot access %s" % species_dir)
-            sys.exit(0)
-        return 1
-
-    def set_get_history(self):
+    def get_history(self):
         """
         Create or set the working history to the current species one
 
-        TODO - move to utilities?
-
         :return:
         """
         try:
-            histories = self.instance.histories.get_histories(name=str(self.full_name))
-            self.history_id = histories[0]["id"]
-            logging.info("History for {0}: {1}".format(self.full_name, self.history_id))
+            histories = self.instance.histories.get_histories(name=str(self.genus_species))
+            if len(histories) == 1:
+                self.history_id = histories[0]["id"]
+                logging.debug("History ID set for {0} {1}: {2}".format(self.genus, self.species, self.history_id))
+            else:
+                logging.critical("Multiple histories exists for {1}: {2}".format(self.genus, self.species))
         except IndexError:
-            logging.info("Creating history for %s" % self.full_name)
-            self.instance.histories.create_history(name=str(self.full_name))
-            histories = self.instance.histories.get_histories(name=str(self.full_name))
-            self.history_id = histories[0]["id"]
-            logging.info("History for {0}: {1}".format(self.full_name, self.history_id))
+            logging.info("Creating history for {0} {1}".format(self.genus, self.species))
+            hist_dict = self.instance.histories.create_history(name=str(self.genus_species))
+            self.history_id = hist_dict["id"]
+            logging.debug("History ID set for {0} {1}: {2}".format(self.genus, self.species, self.history_id))
 
         return self.history_id
 
@@ -82,26 +67,26 @@ class LoadData(speciesData.SpeciesData):
         Will do nothing if H. sapiens isn't in the database
 
         """
-        logging.debug("Getting 'Homo sapiens' ID in instance's chado database")
-        get_sapiens_id_job = self.instance.tools.run_tool(
-            tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_organism_get_organisms/organism_get_organisms/2.3.4+galaxy0",
+
+        logging.debug("Getting 'Homo sapiens' ID in chado database")
+        get_sapiens_id_job_output_dataset_id = utilities.run_tool_and_get_single_output_dataset_id(
+            self.instance,
+            tool_id=constants.GET_ORGANISMS_TOOL, # If this version if not found, Galaxy will use the one that is found
             history_id=self.history_id,
             tool_inputs={"genus": "Homo", "species": "sapiens"})
-        get_sapiens_id_job_output = get_sapiens_id_job["outputs"][0]["id"]
-        get_sapiens_id_json_output = self.instance.datasets.download_dataset(dataset_id=get_sapiens_id_job_output)
+        get_sapiens_id_json_output = self.instance.datasets.download_dataset(dataset_id=get_sapiens_id_job_output_dataset_id)
+
+        logging.info("Deleting Homo 'sapiens' in the instance's chado database")
         try:
-            logging.debug("Deleting Homo 'sapiens' in the instance's chado database")
             get_sapiens_id_final_output = json.loads(get_sapiens_id_json_output)[0]
-            sapiens_id = str(
-                get_sapiens_id_final_output["organism_id"])  # needs to be str to be recognized by the chado tool
-            self.instance.tools.run_tool(
-                tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_organism_delete_organisms/organism_delete_organisms/2.3.4+galaxy0",
+            sapiens_id = str(get_sapiens_id_final_output["organism_id"])  # needs to be str to be recognized by the chado tool
+            utilities.run_tool(
+                self.instance,
+                tool_id=constants.DELETE_ORGANISMS_TOOL,
                 history_id=self.history_id,
-                tool_inputs={"organism": str(sapiens_id)})
-        except bioblend.ConnectionError:
-            logging.debug("Homo sapiens isn't in the instance's chado database (bioblend.ConnectionError)")
+                tool_inputs={"organism": sapiens_id})
         except IndexError:
-            logging.debug("Homo sapiens isn't in the instance's chado database (IndexError)")
+            logging.error("Homo sapiens isn't in the instance's chado database (IndexError)")
             pass
 
     def purge_histories(self):
@@ -114,7 +99,6 @@ class LoadData(speciesData.SpeciesData):
         """
 
         histories = self.instance.histories.get_histories()
-        self.instance.histories.get_histories(deleted=False)
         for h in histories:
             self.instance.histories.delete_history(history_id=h["id"])
 
@@ -128,38 +112,40 @@ class LoadData(speciesData.SpeciesData):
         :return:
         """
 
-        self.goto_species_dir()
+        data_dir_root=os.path.join(self.get_species_dir(), constants.HOST_DATA_DIR)
 
-        # Delete pre-existing lib (probably created by a previous call)
-        gio = GalaxyInstance(url=self.instance_url,
-                             email=self.config["galaxy_default_admin_email"],
-                             password=self.config["galaxy_default_admin_password"])
+        instance = GalaxyInstance(url=self.instance_url,
+                                              email=self.config[constants.CONF_GALAXY_DEFAULT_ADMIN_EMAIL],
+                                              password=self.config[constants.CONF_GALAXY_DEFAULT_ADMIN_PASSWORD]
+                                              )
 
+        logging.info("Looking for project data in %s" % data_dir_root)
         folders = dict()
         post_renaming = {}
 
-        for root, dirs, files in os.walk("./src_data", followlinks=True):
+        for root, dirs, files in os.walk(data_dir_root, followlinks=True):
             file_list = [os.path.join(root, filename) for filename in files]
             folders[root] = file_list
 
         if folders:
             # Delete pre-existing lib (probably created by a previous call)
-            existing = gio.libraries.get_previews(name='Project Data')
+            existing = instance.libraries.get_previews(name='Project Data')
             for lib in existing:
                 if not lib.deleted:
                     logging.info('Pre-existing "Project Data" library %s found, removing it' % lib.id)
-                    gio.libraries.delete(lib.id)
+                    instance.libraries.delete(lib.id)
 
             logging.info("Creating new 'Project Data' library")
-            prj_lib = gio.libraries.create('Project Data', 'Data for current genome annotation project')
+            prj_lib = instance.libraries.create('Project Data', 'Data for current genome annotation project')
             self.library_id = prj_lib.id  # project data folder/library
             logging.info("Library for {0}: {1}".format(self.full_name, self.library_id))
 
             for fname, files in folders.items():
                 if fname and files:
-                    folder_name = fname[len("./src_data") + 1:]
+                    folder_name = re.sub(data_dir_root + "/", "", fname)
                     logging.info("Creating folder: %s" % folder_name)
                     folder = self.create_deep_folder(prj_lib, folder_name)
+
                     for single_file in files:
                         ftype = 'auto'
 
@@ -198,11 +184,16 @@ class LoadData(speciesData.SpeciesData):
                             logging.info("Skipping useless file '%s'" % single_file)
                             continue
 
-                        logging.info("Adding file '%s' with type '%s' and name '%s'" % (single_file, ftype, clean_name))
-                        datasets = prj_lib.upload_from_local(
-                            path=single_file,
+                        single_file_relative_path = re.sub(data_dir_root, constants.CONTAINER_DATA_DIR_ROOT, single_file)
+                        single_file_path_in_container=os.path.join(constants.CONTAINER_DATA_DIR_ROOT, single_file_relative_path)
+
+                        logging.info("Adding file '%s' with type '%s' and name '%s'" % (single_file_path_in_container, ftype, clean_name))
+                        datasets = prj_lib.upload_from_galaxy_fs(
+                            single_file_path_in_container,
                             folder=folder,
-                            file_type=ftype
+                            link_data_only='link_to_files',
+                            file_type=ftype,
+                            tag_using_filenames=False
                         )
 
                         # Rename dataset
@@ -214,10 +205,10 @@ class LoadData(speciesData.SpeciesData):
 
                     time.sleep(1)
 
-        # Wait for uploads to complete
-        logging.info("Waiting for import jobs to finish... please wait")
-
-        # Checking job state (only necessary if ran using SLURM)
+        # # Wait for uploads to complete
+        # logging.info("Waiting for import jobs to finish... please wait")
+        #
+        # # Checking job state (only necessary if ran using SLURM)
         # while True:
         #     try:
         #         # "C" state means the job is completed, no need to wait for it
@@ -231,8 +222,8 @@ class LoadData(speciesData.SpeciesData):
         #             break
         #         else:
         #             raise
-
-        time.sleep(10)
+        #
+        # time.sleep(10)
 
         # Batch renaming --> Throws a critical error at the moment
         # logging.info("Import finished, now renaming datasets with pretty names")
@@ -267,7 +258,29 @@ class LoadData(speciesData.SpeciesData):
 
         return new_folder
 
-    def connect_to_instance(self):
+    def get_bam_label(self, dirname, bam_file):
+
+        bam_id = bam_file
+        if bam_id.endswith('.bam'):
+            bam_id = bam_id[:-4]
+
+        if dirname in self.bam_metadata_cache:
+            if bam_id in self.bam_metadata_cache[dirname] and 'label' in self.bam_metadata_cache[dirname][bam_id] and self.bam_metadata_cache[dirname][bam_id]['label']:
+                return self.bam_metadata_cache[dirname][bam_id]['label']
+            else:
+                return None
+        else:
+            meta_file = os.path.join(dirname, 'metadata.yml')
+            if os.path.exists(meta_file):
+                with open(meta_file) as f:
+                    self.bam_metadata_cache[dirname] = yaml.safe_load(f)
+                    logging.info("Found metadata in %s " % meta_file)
+            else:
+                self.bam_metadata_cache[dirname] = {}
+                logging.info("Did not find metadata in %s " % meta_file)
+            return self.get_bam_label(dirname, bam_file)
+
+    def create_galaxy_instance(self):
         """
         Test the connection to the galaxy instance for the current organism
         Exit if we cannot connect to the instance
@@ -276,10 +289,9 @@ class LoadData(speciesData.SpeciesData):
 
         logging.info("Connecting to the galaxy instance (%s)" % self.instance_url)
         self.instance = galaxy.GalaxyInstance(url=self.instance_url,
-                                              email=self.config["galaxy_default_admin_email"],
-                                              password=self.config["galaxy_default_admin_password"]
+                                              email=self.config[constants.CONF_GALAXY_DEFAULT_ADMIN_EMAIL],
+                                              password=self.config[constants.CONF_GALAXY_DEFAULT_ADMIN_PASSWORD]
                                               )
-        self.instance.histories.get_histories()
 
         try:
             self.instance.histories.get_histories()
@@ -289,26 +301,11 @@ class LoadData(speciesData.SpeciesData):
         else:
             logging.info("Successfully connected to galaxy instance (%s) " % self.instance_url)
 
-
-
-
-def get_species_to_load(sp_dict_list):
-    """
-    """
-
-    
-
-    utilities.get_unique_species_list(sp_dict_list)
-
-
-    return 1
+        return self.instance
 
     
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Automatic data loading in containers and interaction "
-                                                 "with galaxy instances for GGA"
-                                                 ", following the protocol @ "
-                                                 "http://gitlab.sb-roscoff.fr/abims/e-infra/gga")
+    parser = argparse.ArgumentParser(description="Load data into Galaxy library")
 
     parser.add_argument("input",
                         type=str,
@@ -316,11 +313,11 @@ if __name__ == "__main__":
 
     parser.add_argument("-v", "--verbose",
                         help="Increase output verbosity",
-                        action="store_false")
+                        action="store_true")
 
     parser.add_argument("--config",
                         type=str,
-                        help="Config path, default to the 'config' file inside the script repository")
+                        help="Config path, default to 'examples/config.yml'")
 
     parser.add_argument("--main-directory",
                         type=str,
@@ -332,66 +329,61 @@ if __name__ == "__main__":
         logging.basicConfig(level=logging.DEBUG)
     else:
         logging.basicConfig(level=logging.INFO)
-    logging.getLogger("urllib3").setLevel(logging.WARNING)
 
     # Parsing the config file if provided, using the default config otherwise
-    if not args.config:
-        args.config = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "config")
+    if args.config:
+        config_file = os.path.abspath(args.config)
     else:
-        args.config = os.path.abspath(args.config)
+        config_file = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), constants.DEFAULT_CONFIG)
+    config = utilities.parse_config(config_file)
 
+    main_dir = None
     if not args.main_directory:
-        args.main_directory = os.getcwd()
+        main_dir = os.getcwd()
     else:
-        args.main_directory = os.path.abspath(args.main_directory)
+        main_dir = os.path.abspath(args.main_directory)
 
     sp_dict_list = utilities.parse_input(args.input)
 
     unique_sp_dict_list = utilities.get_unique_species_dict_list(sp_dict_list=sp_dict_list)
 
-
     for sp_dict in unique_sp_dict_list:
 
         # Creating an instance of load_data_for_current_species object
         load_data_for_current_species = LoadData(parameters_dictionary=sp_dict)
 
         # Starting
-        logging.info("gga_load_data.py called for %s" % load_data_for_current_species.full_name)
+        logging.info("gga_load_data.py called for {0} {1}".format(load_data_for_current_species.genus, load_data_for_current_species.species))
 
         # Setting some of the instance attributes
-        load_data_for_current_species.main_dir = args.main_directory
+        load_data_for_current_species.main_dir = main_dir
         load_data_for_current_species.species_dir = os.path.join(load_data_for_current_species.main_dir,
                                                                  load_data_for_current_species.genus_species +
                                                                  "/")
 
         # Parse the config yaml file
-        load_data_for_current_species.config = utilities.parse_config(args.config)
+        load_data_for_current_species.config = config
         # Set the instance url attribute -- Does not work with localhost on scratch (ALB)
         load_data_for_current_species.instance_url = "http://localhost:{0}/sp/{1}_{2}/galaxy/".format(
-                load_data_for_current_species.config["http_port"],
+                load_data_for_current_species.config[constants.CONF_ALL_HTTP_PORT],
                 load_data_for_current_species.genus_lowercase,
                 load_data_for_current_species.species)
 
-        
-
         # Check the galaxy container state and proceed if the galaxy services are up and running
         if utilities.check_galaxy_state(genus_lowercase=load_data_for_current_species.genus_lowercase,
                                         species=load_data_for_current_species.species,
                                         script_dir=load_data_for_current_species.script_dir):
 
-            # Load config file
-            load_data_for_current_species.config = utilities.parse_config(args.config)
-
-            # Testing connection to the instance
-            load_data_for_current_species.connect_to_instance()
+            # Create the Galaxy instance
+            load_data_for_current_species.instance = load_data_for_current_species.create_galaxy_instance()
 
             # Load the datasets into a galaxy library
-            logging.info("Setting up library for %s" % load_data_for_current_species.full_name)
+            logging.info("Setting up library for {0} {1}".format(load_data_for_current_species.genus, load_data_for_current_species.species))
             load_data_for_current_species.setup_library()
-            logging.info("Successfully set up library in galaxy for %s" % load_data_for_current_species.full_name)
+            logging.debug("Successfully set up library in galaxy for {0} {1}".format(load_data_for_current_species.genus, load_data_for_current_species.species))
 
             # Set or get the history for the current organism
-            load_data_for_current_species.set_get_history()
+            load_data_for_current_species.get_history()
             
             # Remove H. sapiens from database if here
             # TODO: set a dedicated history for removing H. sapiens (instead of doing it into a species history)
@@ -399,11 +391,10 @@ if __name__ == "__main__":
 
             # logging.info("Importing datasets into history for %s" % load_data_for_current_species.full_name)
             # load_data_for_current_species.import_datasets_into_history()  # Option "--load-history"
-
             # load_data_for_current_species.purge_histories()  # Testing purposes
 
-            logging.info("Data successfully loaded and imported for %s" % load_data_for_current_species.full_name)
+            logging.info("Data successfully loaded and imported for {0} {1}".format(load_data_for_current_species.genus, load_data_for_current_species.species))
 
         else:
-            logging.critical("The galaxy container for %s is not ready yet!" % load_data_for_current_species.full_name)
+            logging.critical("The galaxy container for {0} {1} is not ready yet".format(load_data_for_current_species.genus, load_data_for_current_species.species))
             sys.exit()