Release 2.0

gga_load_data.py:

• Factorize goto_species_dir(), create_galaxy_instance(), run_tool(), get_species_dir()
• Move constants at the beginning
• Revert upload_from_local() to upload_from_galaxy_fs() (to enable symlink, instead of copy)
• Add missing get_bam_label()

README:

• update contributors

gga_load_data.py

 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
+import re
 
 import bioblend
 import argparse
@@ -9,8 +10,10 @@ import sys
 import time
 import json
 import yaml
+import subprocess
 
 from bioblend import galaxy
+from bioblend.galaxy.objects import GalaxyInstance
 
 import utilities
 import speciesData
@@ -27,6 +30,8 @@ Do not call this script before the galaxy container is ready
 GET_ORGANISMS_TOOL = "toolshed.g2.bx.psu.edu/repos/gga/chado_organism_get_organisms/organism_get_organisms/2.3.4+galaxy0"
 DELETE_ORGANISMS_TOOL = "toolshed.g2.bx.psu.edu/repos/gga/chado_organism_delete_organisms/organism_delete_organisms/2.3.4+galaxy0"
 
+HOST_DATA_DIR='src_data'
+CONTAINER_DATA_DIR_ROOT='/project_data'
 
 class LoadData(speciesData.SpeciesData):
     """
@@ -100,7 +105,7 @@ class LoadData(speciesData.SpeciesData):
 
         return histories
 
-    def setup_library(self, instance):
+    def setup_library(self):
         """
         Create a "Project Data" library in galaxy, mirroring the "src_data" folder of the current organism
         directory tree
@@ -108,12 +113,18 @@ class LoadData(speciesData.SpeciesData):
         :return:
         """
 
-        self.goto_species_dir()
+        data_dir_root=os.path.join(self.get_species_dir(), HOST_DATA_DIR)
 
+        instance = GalaxyInstance(url=self.instance_url,
+                                              email=self.config["galaxy_default_admin_email"],
+                                              password=self.config["galaxy_default_admin_password"]
+                                              )
+
+        logging.info("Looking for project data in %s" % data_dir_root)
         folders = dict()
         post_renaming = {}
 
-        for root, dirs, files in os.walk("./src_data", followlinks=True):
+        for root, dirs, files in os.walk(data_dir_root, followlinks=True):
             file_list = [os.path.join(root, filename) for filename in files]
             folders[root] = file_list
 
@@ -132,7 +143,7 @@ class LoadData(speciesData.SpeciesData):
 
             for fname, files in folders.items():
                 if fname and files:
-                    folder_name = fname[len("./src_data") + 1:]
+                    folder_name = re.sub(data_dir_root + "/", "", fname)
                     logging.info("Creating folder: %s" % folder_name)
                     folder = self.create_deep_folder(prj_lib, folder_name)
 
@@ -174,9 +185,12 @@ class LoadData(speciesData.SpeciesData):
                             logging.info("Skipping useless file '%s'" % single_file)
                             continue
 
-                        logging.info("Adding file '%s' with type '%s' and name '%s'" % (single_file, ftype, clean_name))
+                        single_file_relative_path = re.sub(data_dir_root, CONTAINER_DATA_DIR_ROOT, single_file)
+                        single_file_path_in_container=os.path.join(CONTAINER_DATA_DIR_ROOT, single_file_relative_path)
+
+                        logging.info("Adding file '%s' with type '%s' and name '%s'" % (single_file_path_in_container, ftype, clean_name))
                         datasets = prj_lib.upload_from_galaxy_fs(
-                            single_file,
+                            single_file_path_in_container,
                             folder=folder,
                             link_data_only='link_to_files',
                             file_type=ftype,
@@ -192,10 +206,10 @@ class LoadData(speciesData.SpeciesData):
 
                     time.sleep(1)
 
-        # Wait for uploads to complete
-        logging.info("Waiting for import jobs to finish... please wait")
-
-        # Checking job state (only necessary if ran using SLURM)
+        # # Wait for uploads to complete
+        # logging.info("Waiting for import jobs to finish... please wait")
+        #
+        # # Checking job state (only necessary if ran using SLURM)
         # while True:
         #     try:
         #         # "C" state means the job is completed, no need to wait for it
@@ -209,8 +223,8 @@ class LoadData(speciesData.SpeciesData):
         #             break
         #         else:
         #             raise
-
-        time.sleep(10)
+        #
+        # time.sleep(10)
 
         # Batch renaming --> Throws a critical error at the moment
         # logging.info("Import finished, now renaming datasets with pretty names")
@@ -365,7 +379,7 @@ if __name__ == "__main__":
 
             # Load the datasets into a galaxy library
             logging.info("Setting up library for {0} {1}".format(load_data_for_current_species.genus, load_data_for_current_species.species))
-            load_data_for_current_species.setup_library(load_data_for_current_species.instance)
+            load_data_for_current_species.setup_library()
             logging.debug("Successfully set up library in galaxy for {0} {1}".format(load_data_for_current_species.genus, load_data_for_current_species.species))
 
             # Set or get the history for the current organism