Skip to content
Snippets Groups Projects

Release 2.0

Merged Release 2.0
release_2.0 into dev
Merged Loraine Gueguen requested to merge release_2.0 into dev
Viewing commit 73cc89a8
Prev Next
Show latest version
1 file
+ 67
− 90
Compare changes
  • Side-by-side
  • Inline
gga_load_data.py
+ 93
− 106
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import re
import bioblend
import argparse
import os
import subprocess
import logging
import sys
import fnmatch
import time
import json
import re
import stat
import shutil
import yaml
import subprocess
from bioblend.galaxy.objects import GalaxyInstance
from bioblend import galaxy
from bioblend.galaxy.objects import GalaxyInstance
import utilities
import speciesData
@@ -23,11 +21,17 @@ import speciesData
"""
gga_load_data.py
Usage: $ python3 gga_init.py -i input_example.yml --config config.yml [OPTIONS]
Usage: $ python3 gga_load_data.py -i input_example.yml --config config.yml [OPTIONS]
Do not call this script before the galaxy container is ready
"""
# If this version if not found, Galaxy will use the one that is found
GET_ORGANISMS_TOOL = "toolshed.g2.bx.psu.edu/repos/gga/chado_organism_get_organisms/organism_get_organisms/2.3.4+galaxy0"
DELETE_ORGANISMS_TOOL = "toolshed.g2.bx.psu.edu/repos/gga/chado_organism_delete_organisms/organism_delete_organisms/2.3.4+galaxy0"
HOST_DATA_DIR='src_data'
CONTAINER_DATA_DIR_ROOT='/project_data'
class LoadData(speciesData.SpeciesData):
"""
@@ -39,23 +43,7 @@ class LoadData(speciesData.SpeciesData):
"""
def goto_species_dir(self):
"""
Go to the species directory (starting from the main dir)
:return:
"""
os.chdir(self.main_dir)
species_dir = os.path.join(self.main_dir, self.genus_species) + "/"
try:
os.chdir(species_dir)
except OSError:
logging.critical("Cannot access %s" % species_dir)
sys.exit(0)
return 1
def set_get_history(self):
def get_history(self):
"""
Create or set the working history to the current species one
@@ -63,13 +51,15 @@ class LoadData(speciesData.SpeciesData):
"""
try:
histories = self.instance.histories.get_histories(name=str(self.genus_species))
self.history_id = histories[0]["id"]
logging.debug("History ID set for {0} {1}: {2}".format(self.genus, self.species, self.history_id))
if len(histories) == 1:
self.history_id = histories[0]["id"]
logging.debug("History ID set for {0} {1}: {2}".format(self.genus, self.species, self.history_id))
else:
logging.critical("Multiple histories exists for {1}: {2}".format(self.genus, self.species))
except IndexError:
logging.info("Creating history for {0} {1}".format(self.genus, self.species))
self.instance.histories.create_history(name=str(self.genus_species))
histories = self.instance.histories.get_histories(name=str(self.genus_species))
self.history_id = histories[0]["id"]
hist_dict = self.instance.histories.create_history(name=str(self.genus_species))
self.history_id = hist_dict["id"]
logging.debug("History ID set for {0} {1}: {2}".format(self.genus, self.species, self.history_id))
return self.history_id
@@ -81,32 +71,23 @@ class LoadData(speciesData.SpeciesData):
"""
get_organism_tool = self.instance.tools.show_tool("toolshed.g2.bx.psu.edu/repos/gga/chado_organism_get_organisms/organism_get_organisms/2.3.4+galaxy0")
delete_organism_tool = self.instance.tools.show_tool("toolshed.g2.bx.psu.edu/repos/gga/chado_organism_delete_organisms/organism_delete_organisms/2.3.4+galaxy0")
get_organism_tool_version = get_organism_tool["version"]
delete_organism_tool_version = delete_organism_tool["version"]
logging.debug("Getting 'Homo sapiens' ID in instance's chado database")
get_sapiens_id_job = self.instance.tools.run_tool(
tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_organism_get_organisms/organism_get_organisms/%s" % get_organism_tool_version,
logging.debug("Getting 'Homo sapiens' ID in chado database")
get_sapiens_id_job_output_dataset_id = utilities.run_tool_and_get_single_output_dataset_id(self.instance,
tool_id=GET_ORGANISMS_TOOL,
history_id=self.history_id,
tool_inputs={"genus": "Homo", "species": "sapiens"})
get_sapiens_id_job_output = get_sapiens_id_job["outputs"][0]["id"]
get_sapiens_id_json_output = self.instance.datasets.download_dataset(dataset_id=get_sapiens_id_job_output)
get_sapiens_id_json_output = self.instance.datasets.download_dataset(dataset_id=get_sapiens_id_job_output_dataset_id)
logging.debug("Deleting Homo 'sapiens' in the instance's chado database")
try:
logging.debug("Deleting Homo 'sapiens' in the instance's chado database")
get_sapiens_id_final_output = json.loads(get_sapiens_id_json_output)[0]
sapiens_id = str(
get_sapiens_id_final_output["organism_id"]) # needs to be str to be recognized by the chado tool
self.instance.tools.run_tool(
tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_organism_delete_organisms/organism_delete_organisms/%s" % delete_organism_tool_version,
sapiens_id = str(get_sapiens_id_final_output["organism_id"]) # needs to be str to be recognized by the chado tool
utilities.run_tool(
tool_id=DELETE_ORGANISMS_TOOL,
history_id=self.history_id,
tool_inputs={"organism": str(sapiens_id)})
except bioblend.ConnectionError:
logging.debug("Homo sapiens isn't in the instance's chado database (bioblend.ConnectionError)")
tool_inputs={"organism": sapiens_id})
except IndexError:
logging.debug("Homo sapiens isn't in the instance's chado database (IndexError)")
logging.error("Homo sapiens isn't in the instance's chado database (IndexError)")
pass
def purge_histories(self):
@@ -119,7 +100,6 @@ class LoadData(speciesData.SpeciesData):
"""
histories = self.instance.histories.get_histories()
self.instance.histories.get_histories(deleted=False)
for h in histories:
self.instance.histories.delete_history(history_id=h["id"])
@@ -133,38 +113,40 @@ class LoadData(speciesData.SpeciesData):
:return:
"""
self.goto_species_dir()
data_dir_root=os.path.join(self.get_species_dir(), HOST_DATA_DIR)
# Delete pre-existing lib (probably created by a previous call)
gio = GalaxyInstance(url=self.instance_url,
email=self.config["galaxy_default_admin_email"],
password=self.config["galaxy_default_admin_password"])
instance = GalaxyInstance(url=self.instance_url,
email=self.config["galaxy_default_admin_email"],
password=self.config["galaxy_default_admin_password"]
)
logging.info("Looking for project data in %s" % data_dir_root)
folders = dict()
post_renaming = {}
for root, dirs, files in os.walk("./src_data", followlinks=True):
for root, dirs, files in os.walk(data_dir_root, followlinks=True):
file_list = [os.path.join(root, filename) for filename in files]
folders[root] = file_list
if folders:
# Delete pre-existing lib (probably created by a previous call)
existing = gio.libraries.get_previews(name='Project Data')
existing = instance.libraries.get_previews(name='Project Data')
for lib in existing:
if not lib.deleted:
logging.info('Pre-existing "Project Data" library %s found, removing it' % lib.id)
gio.libraries.delete(lib.id)
instance.libraries.delete(lib.id)
logging.info("Creating new 'Project Data' library")
prj_lib = gio.libraries.create('Project Data', 'Data for current genome annotation project')
prj_lib = instance.libraries.create('Project Data', 'Data for current genome annotation project')
self.library_id = prj_lib.id # project data folder/library
logging.info("Library for {0}: {1}".format(self.full_name, self.library_id))
for fname, files in folders.items():
if fname and files:
folder_name = fname[len("./src_data") + 1:]
folder_name = re.sub(data_dir_root + "/", "", fname)
logging.info("Creating folder: %s" % folder_name)
folder = self.create_deep_folder(prj_lib, folder_name)
for single_file in files:
ftype = 'auto'
@@ -203,11 +185,16 @@ class LoadData(speciesData.SpeciesData):
logging.info("Skipping useless file '%s'" % single_file)
continue
logging.info("Adding file '%s' with type '%s' and name '%s'" % (single_file, ftype, clean_name))
datasets = prj_lib.upload_from_local(
path=single_file,
single_file_relative_path = re.sub(data_dir_root, CONTAINER_DATA_DIR_ROOT, single_file)
single_file_path_in_container=os.path.join(CONTAINER_DATA_DIR_ROOT, single_file_relative_path)
logging.info("Adding file '%s' with type '%s' and name '%s'" % (single_file_path_in_container, ftype, clean_name))
datasets = prj_lib.upload_from_galaxy_fs(
single_file_path_in_container,
folder=folder,
file_type=ftype
link_data_only='link_to_files',
file_type=ftype,
tag_using_filenames=False
)
# Rename dataset
@@ -219,10 +206,10 @@ class LoadData(speciesData.SpeciesData):
time.sleep(1)
# Wait for uploads to complete
logging.info("Waiting for import jobs to finish... please wait")
# Checking job state (only necessary if ran using SLURM)
# # Wait for uploads to complete
# logging.info("Waiting for import jobs to finish... please wait")
#
# # Checking job state (only necessary if ran using SLURM)
# while True:
# try:
# # "C" state means the job is completed, no need to wait for it
@@ -236,8 +223,8 @@ class LoadData(speciesData.SpeciesData):
# break
# else:
# raise
time.sleep(10)
#
# time.sleep(10)
# Batch renaming --> Throws a critical error at the moment
# logging.info("Import finished, now renaming datasets with pretty names")
@@ -272,7 +259,29 @@ class LoadData(speciesData.SpeciesData):
return new_folder
def connect_to_instance(self):
def get_bam_label(self, dirname, bam_file):
bam_id = bam_file
if bam_id.endswith('.bam'):
bam_id = bam_id[:-4]
if dirname in self.bam_metadata_cache:
if bam_id in self.bam_metadata_cache[dirname] and 'label' in self.bam_metadata_cache[dirname][bam_id] and self.bam_metadata_cache[dirname][bam_id]['label']:
return self.bam_metadata_cache[dirname][bam_id]['label']
else:
return None
else:
meta_file = os.path.join(dirname, 'metadata.yml')
if os.path.exists(meta_file):
with open(meta_file) as f:
self.bam_metadata_cache[dirname] = yaml.safe_load(f)
logging.info("Found metadata in %s " % meta_file)
else:
self.bam_metadata_cache[dirname] = {}
logging.info("Did not find metadata in %s " % meta_file)
return self.get_bam_label(dirname, bam_file)
def create_galaxy_instance(self):
"""
Test the connection to the galaxy instance for the current organism
Exit if we cannot connect to the instance
@@ -284,7 +293,6 @@ class LoadData(speciesData.SpeciesData):
email=self.config["galaxy_default_admin_email"],
password=self.config["galaxy_default_admin_password"]
)
self.instance.histories.get_histories()
try:
self.instance.histories.get_histories()
@@ -294,19 +302,7 @@ class LoadData(speciesData.SpeciesData):
else:
logging.info("Successfully connected to galaxy instance (%s) " % self.instance_url)
def get_species_to_load(sp_dict_list):
"""
"""
utilities.get_unique_species_list(sp_dict_list)
return 1
return self.instance
if __name__ == "__main__":
@@ -338,64 +334,56 @@ if __name__ == "__main__":
else:
logging.basicConfig(level=logging.INFO)
# Parsing the config file if provided, using the default config otherwise
if not args.config:
args.config = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "config")
else:
args.config = os.path.abspath(args.config)
config_file = os.path.abspath(args.config)
config = utilities.parse_config(config_file)
main_dir = None
if not args.main_directory:
args.main_directory = os.getcwd()
main_dir = os.getcwd()
else:
args.main_directory = os.path.abspath(args.main_directory)
main_dir = os.path.abspath(args.main_directory)
sp_dict_list = utilities.parse_input(args.input)
unique_sp_dict_list = utilities.get_unique_species_dict_list(sp_dict_list=sp_dict_list)
for sp_dict in unique_sp_dict_list:
# Creating an instance of load_data_for_current_species object
load_data_for_current_species = LoadData(parameters_dictionary=sp_dict)
# Starting
logging.info("gga_load_data.py called for %s" % load_data_for_current_species.full_name)
logging.info("gga_load_data.py called for {0} {1}".format(load_data_for_current_species.genus, load_data_for_current_species.species))
# Setting some of the instance attributes
load_data_for_current_species.main_dir = args.main_directory
load_data_for_current_species.main_dir = main_dir
load_data_for_current_species.species_dir = os.path.join(load_data_for_current_species.main_dir,
load_data_for_current_species.genus_species +
"/")
# Parse the config yaml file
load_data_for_current_species.config = utilities.parse_config(args.config)
load_data_for_current_species.config = config
# Set the instance url attribute -- Does not work with localhost on scratch (ALB)
load_data_for_current_species.instance_url = "http://localhost:{0}/sp/{1}_{2}/galaxy/".format(
load_data_for_current_species.config["http_port"],
load_data_for_current_species.genus_lowercase,
load_data_for_current_species.species)
# Check the galaxy container state and proceed if the galaxy services are up and running
if utilities.check_galaxy_state(genus_lowercase=load_data_for_current_species.genus_lowercase,
species=load_data_for_current_species.species,
script_dir=load_data_for_current_species.script_dir):
# Load config file
load_data_for_current_species.config = utilities.parse_config(args.config)
# Testing connection to the instance
load_data_for_current_species.connect_to_instance()
# Create the Galaxy instance
load_data_for_current_species.instance = load_data_for_current_species.create_galaxy_instance()
# Load the datasets into a galaxy library
logging.info("Setting up library for {0} {1}".format(load_data_for_current_species.genus, load_data_for_current_species.species))
load_data_for_current_species.setup_library()
logging.info("Successfully set up library in galaxy for {0} {1}".format(load_data_for_current_species.genus, load_data_for_current_species.species))
logging.debug("Successfully set up library in galaxy for {0} {1}".format(load_data_for_current_species.genus, load_data_for_current_species.species))
# Set or get the history for the current organism
load_data_for_current_species.set_get_history()
load_data_for_current_species.get_history()
# Remove H. sapiens from database if here
# TODO: set a dedicated history for removing H. sapiens (instead of doing it into a species history)
@@ -403,7 +391,6 @@ if __name__ == "__main__":
# logging.info("Importing datasets into history for %s" % load_data_for_current_species.full_name)
# load_data_for_current_species.import_datasets_into_history() # Option "--load-history"
# load_data_for_current_species.purge_histories() # Testing purposes
logging.info("Data successfully loaded and imported for {0} {1}".format(load_data_for_current_species.genus, load_data_for_current_species.species))