Release 2.0

gga_load_data.py:

• Factorize goto_species_dir(), create_galaxy_instance(), run_tool(), get_species_dir()
• Move constants at the beginning
• Revert upload_from_local() to upload_from_galaxy_fs() (to enable symlink, instead of copy)
• Add missing get_bam_label()

README:

• update contributors

gga_load_data.py

 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
+import re
 
 import bioblend
 import argparse
@@ -9,6 +10,7 @@ import sys
 import time
 import json
 import yaml
+import subprocess
 
 from bioblend import galaxy
 from bioblend.galaxy.objects import GalaxyInstance
@@ -28,6 +30,8 @@ Do not call this script before the galaxy container is ready
 GET_ORGANISMS_TOOL = "toolshed.g2.bx.psu.edu/repos/gga/chado_organism_get_organisms/organism_get_organisms/2.3.4+galaxy0"
 DELETE_ORGANISMS_TOOL = "toolshed.g2.bx.psu.edu/repos/gga/chado_organism_delete_organisms/organism_delete_organisms/2.3.4+galaxy0"
 
+HOST_DATA_DIR='src_data'
+CONTAINER_DATA_DIR_ROOT='/project_data'
 
 class LoadData(speciesData.SpeciesData):
     """
@@ -101,7 +105,7 @@ class LoadData(speciesData.SpeciesData):
 
         return histories
 
-    def setup_library(self, instance, data_dir_root='./src_data'):
+    def setup_library(self):
         """
         Create a "Project Data" library in galaxy, mirroring the "src_data" folder of the current organism
         directory tree
@@ -109,9 +113,9 @@ class LoadData(speciesData.SpeciesData):
         :return:
         """
 
-        self.goto_species_dir()
+        data_dir_root=os.path.join(self.get_species_dir(), HOST_DATA_DIR)
 
-        gio = GalaxyInstance(url=self.instance_url,
+        instance = GalaxyInstance(url=self.instance_url,
                                               email=self.config["galaxy_default_admin_email"],
                                               password=self.config["galaxy_default_admin_password"]
                                               )
@@ -139,7 +143,7 @@ class LoadData(speciesData.SpeciesData):
 
             for fname, files in folders.items():
                 if fname and files:
-                    folder_name = fname[len(data_dir_root) + 1:]
+                    folder_name = re.sub(data_dir_root + "/", "", fname)
                     logging.info("Creating folder: %s" % folder_name)
                     folder = self.create_deep_folder(prj_lib, folder_name)
 
@@ -181,9 +185,12 @@ class LoadData(speciesData.SpeciesData):
                             logging.info("Skipping useless file '%s'" % single_file)
                             continue
 
-                        logging.info("Adding file '%s' with type '%s' and name '%s'" % (single_file, ftype, clean_name))
+                        single_file_relative_path = re.sub(data_dir_root, CONTAINER_DATA_DIR_ROOT, single_file)
+                        single_file_path_in_container=os.path.join(CONTAINER_DATA_DIR_ROOT, single_file_relative_path)
+
+                        logging.info("Adding file '%s' with type '%s' and name '%s'" % (single_file_path_in_container, ftype, clean_name))
                         datasets = prj_lib.upload_from_galaxy_fs(
-                            single_file,
+                            single_file_path_in_container,
                             folder=folder,
                             link_data_only='link_to_files',
                             file_type=ftype,
@@ -203,19 +210,19 @@ class LoadData(speciesData.SpeciesData):
         logging.info("Waiting for import jobs to finish... please wait")
 
         # Checking job state (only necessary if ran using SLURM)
-        # while True:
-        #     try:
-        #         # "C" state means the job is completed, no need to wait for it
-        #         ret = subprocess.check_output("squeue | grep -v \"C debug\" | grep -v \"JOBID\" || true",
-        #                                       shell=True)
-        #         if not len(ret):
-        #             break
-        #         time.sleep(3)
-        #     except subprocess.CalledProcessError as inst:
-        #         if inst.returncode == 153:  # queue is empty
-        #             break
-        #         else:
-        #             raise
+        while True:
+            try:
+                # "C" state means the job is completed, no need to wait for it
+                ret = subprocess.check_output("squeue | grep -v \"C debug\" | grep -v \"JOBID\" || true",
+                                              shell=True)
+                if not len(ret):
+                    break
+                time.sleep(3)
+            except subprocess.CalledProcessError as inst:
+                if inst.returncode == 153:  # queue is empty
+                    break
+                else:
+                    raise
 
         time.sleep(10)
 
@@ -372,7 +379,7 @@ if __name__ == "__main__":
 
             # Load the datasets into a galaxy library
             logging.info("Setting up library for {0} {1}".format(load_data_for_current_species.genus, load_data_for_current_species.species))
-            load_data_for_current_species.setup_library(load_data_for_current_species.instance)
+            load_data_for_current_species.setup_library()
             logging.debug("Successfully set up library in galaxy for {0} {1}".format(load_data_for_current_species.genus, load_data_for_current_species.species))
 
             # Set or get the history for the current organism