run_workflow_phaeoexplorer.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import bioblend
import bioblend.galaxy.objects
import argparse
import os
import logging
import sys
import json
import time

from bioblend.galaxy.objects import GalaxyInstance
from bioblend import galaxy

import utilities
import speciesData

""" 
gga_init.py

Usage: $ python3 gga_init.py -i input_example.yml --config [config file] [OPTIONS]
"""


class RunWorkflow(speciesData.SpeciesData):
    """
    Run a workflow into the galaxy instance's history of a given species


    This script is made to work for a Phaeoexplorer-specific workflow, but can be adapted to run any workflow,
    provided the user creates their own workflow in a .ga format, and change the set_parameters function
    to have the correct parameters for their workflow

    """

    def set_get_history(self):
        """
        Create or set the working history to the current species one

        :return:
        """
        try:
            histories = self.instance.histories.get_histories(name=str(self.genus_species))
            self.history_id = histories[0]["id"]
            logging.debug("History ID set for {0}: {1}".format(self.full_name, self.history_id))
        except IndexError:
            logging.info("Creating history for %s" % self.full_name)
            self.instance.histories.create_history(name=str(self.full_name))
            histories = self.instance.histories.get_histories(name=str(self.genus_species))
            self.history_id = histories[0]["id"]
            logging.debug("History ID set for {0}: {1}".format(self.full_name, self.history_id))

        return self.history_id

    def get_instance_attributes(self):
        """
        retrieves instance attributes:
        - working history ID
        - libraries ID (there should only be one library!)
        - datasets IDs

        :return:
        """

        self.set_get_history()

        logging.debug("History ID: %s" % self.history_id)
        libraries = self.instance.libraries.get_libraries()  # normally only one library
        library_id = self.instance.libraries.get_libraries()[0]["id"]  # project data folder/library
        logging.debug("Library ID: %s" % self.library_id)
        instance_source_data_folders = self.instance.libraries.get_folders(library_id=library_id)

        return {"history_id": self.history_id, "library_id": library_id}


    def connect_to_instance(self):
        """
        Test the connection to the galaxy instance for the current organism
        Exit if we cannot connect to the instance

        """

        # logging.debug("Connecting to the galaxy instance (%s)" % self.instance_url)
        self.instance = galaxy.GalaxyInstance(url=self.instance_url,
                                              email=self.config["galaxy_default_admin_email"],
                                              password=self.config["galaxy_default_admin_password"]
                                              )
        self.instance.histories.get_histories()

        try:
            self.instance.histories.get_histories()
        except bioblend.ConnectionError:
            logging.critical("Cannot connect to galaxy instance (%s)" % self.instance_url)
            sys.exit()
        else:
            # logging.debug("Successfully connected to galaxy instance (%s) " % self.instance_url)
            return 1



    def return_instance(self):


        return self.instance
        


    def install_changesets_revisions_for_individual_tools(self):
        """
        This function is used to verify that installed tools called outside workflows have the correct versions and changesets
        If it finds versions don't match, will install the correct version + changeset in the instance
        Doesn't do anything if versions match
        
        :return:
        """

        self.connect_to_instance()

        logging.info("Validating installed individual tools versions and changesets")

        # Verify that the add_organism and add_analysis versions are correct in the toolshed
        add_organism_tool = self.instance.tools.show_tool("toolshed.g2.bx.psu.edu/repos/gga/chado_organism_add_organism/organism_add_organism/2.3.4+galaxy0")
        add_analysis_tool = self.instance.tools.show_tool("toolshed.g2.bx.psu.edu/repos/gga/chado_analysis_add_analysis/analysis_add_analysis/2.3.4+galaxy0")
        get_organism_tool = self.instance.tools.show_tool("toolshed.g2.bx.psu.edu/repos/gga/chado_organism_get_organisms/organism_get_organisms/2.3.4+galaxy0")
        get_analysis_tool = self.instance.tools.show_tool("toolshed.g2.bx.psu.edu/repos/gga/chado_analysis_get_analyses/analysis_get_analyses/2.3.4+galaxy0")

        # changeset for 2.3.4+galaxy0 has to be manually found because there is no way to get the wanted changeset of a non installed tool via bioblend 
        # except for workflows (.ga) that already contain the changeset revisions inside the steps ids
        
        if get_organism_tool["version"] != "2.3.4+galaxy0":
            toolshed_dict = get_organism_tool["tool_shed_repository"]
            logging.warning("Changeset for %s is not installed" % toolshed_dict["name"])
            changeset_revision = "831229e6cda2"
            name = toolshed_dict["name"]
            owner = toolshed_dict["owner"]
            toolshed = "https://" + toolshed_dict["tool_shed"]
            logging.warning("Installing changeset revision {0} for {1}".format(changeset_revision, name))

            self.instance.toolshed.install_repository_revision(tool_shed_url=toolshed, name=name, owner=owner, 
                                                               changeset_revision=changeset_revision,
                                                               install_tool_dependencies=True,
                                                               install_repository_dependencies=False,
                                                               install_resolver_dependencies=True)

        if get_analysis_tool["version"] != "2.3.4+galaxy0":
            toolshed_dict = changeset_revision["tool_shed_repository"]
            logging.warning("Changeset for %s is not installed" % toolshed_dict["name"])
            changeset_revision = "a867923f555e"
            name = toolshed_dict["name"]
            owner = toolshed_dict["owner"]
            toolshed = "https://" + toolshed_dict["tool_shed"]
            logging.warning("Installing changeset revision {0} for {1}".format(changeset_revision, name))

            self.instance.toolshed.install_repository_revision(tool_shed_url=toolshed, name=name, owner=owner, 
                                                               changeset_revision=changeset_revision,
                                                               install_tool_dependencies=True,
                                                               install_repository_dependencies=False,
                                                               install_resolver_dependencies=True)

        if add_organism_tool["version"] != "2.3.4+galaxy0":
            toolshed_dict = add_organism_tool["tool_shed_repository"]
            logging.warning("Changeset for %s is not installed" % toolshed_dict["name"])
            changeset_revision = "1f12b9650028"
            name = toolshed_dict["name"]
            owner = toolshed_dict["owner"]
            toolshed = "https://" + toolshed_dict["tool_shed"]
            logging.warning("Installing changeset revision {0} for {1}".format(changeset_revision, name))

            self.instance.toolshed.install_repository_revision(tool_shed_url=toolshed, name=name, owner=owner, 
                                                               changeset_revision=changeset_revision,
                                                               install_tool_dependencies=True,
                                                               install_repository_dependencies=False,
                                                               install_resolver_dependencies=True)

        if add_analysis_tool["version"] != "2.3.4+galaxy0":
            toolshed_dict = add_analysis_tool["tool_shed_repository"]
            logging.warning("Changeset for %s is not installed" % toolshed_dict["name"])
            changeset_revision = "10b2b1c70e69"
            name = toolshed_dict["name"]
            owner = toolshed_dict["owner"]
            toolshed = "https://" + toolshed_dict["tool_shed"]
            logging.warning("Installing changeset revision {0} for {1}".format(changeset_revision, name))

            self.instance.toolshed.install_repository_revision(tool_shed_url=toolshed, name=name, owner=owner, 
                                                               changeset_revision=changeset_revision,
                                                               install_tool_dependencies=True,
                                                               install_repository_dependencies=False,
                                                               install_resolver_dependencies=True)


        sync_analysis_tool = self.instance.tools.show_tool("toolshed.g2.bx.psu.edu/repos/gga/tripal_analysis_sync/analysis_sync/3.2.1.0")
        sync_organism_tool = self.instance.tools.show_tool("toolshed.g2.bx.psu.edu/repos/gga/tripal_organism_sync/organism_sync/3.2.1.0")

        if sync_analysis_tool["version"] != "3.2.1.0":
            toolshed_dict = sync_analysis_tool["tool_shed_repository"]
            logging.warning("Changeset for %s is not installed" % toolshed_dict["name"])
            changeset_revision = "f487ff676088"
            name = toolshed_dict["name"]
            owner = toolshed_dict["owner"]
            toolshed = "https://" + toolshed_dict["tool_shed"]
            logging.warning("Installing changeset revision {0} for {1}".format(changeset_revision, name))

            self.instance.toolshed.install_repository_revision(tool_shed_url=toolshed, name=name, owner=owner, 
                                                               changeset_revision=changeset_revision,
                                                               install_tool_dependencies=True,
                                                               install_repository_dependencies=False,
                                                               install_resolver_dependencies=True)

        if sync_organism_tool["version"] != "3.2.1.0":
            toolshed_dict = sync_organism_tool["tool_shed_repository"]
            logging.warning("Changeset for %s is not installed" % toolshed_dict["name"])
            changeset_revision = "afd5d92745fb"
            name = toolshed_dict["name"]
            owner = toolshed_dict["owner"]
            toolshed = "https://" + toolshed_dict["tool_shed"]
            logging.warning("Installing changeset revision {0} for {1}".format(changeset_revision, name))

            self.instance.toolshed.install_repository_revision(tool_shed_url=toolshed, name=name, owner=owner, 
                                                               changeset_revision=changeset_revision,
                                                               install_tool_dependencies=True,
                                                               install_repository_dependencies=False,
                                                               install_resolver_dependencies=True)


        logging.info("Success: individual tools versions and changesets validated")



    def tripal_synchronize_organism_analyses(self):
        """
        """
        show_tool_tripal_sync = self.instance.tools.show_tool(tool_id="toolshed.g2.bx.psu.edu/repos/gga/tripal_organism_sync/organism_sync/3.2.1.0", io_details=True)
        org_sync = "toolshed.g2.bx.psu.edu/repos/gga/tripal_organism_sync/organism_sync/3.2.1.0"
        org_sync = self.instance.tools.run_tool(tool_id="toolshed.g2.bx.psu.edu/repos/gga/tripal_organism_sync/organism_sync/3.2.1.0",
                                                history_id=self.history_id,
                                                tool_inputs={"organism_id": "2"})
        org_sync_job_out = org_sync["outputs"]


    def add_organism_ogs_genome_analyses(self):
        """
        Add OGS and genome vX analyses to Chado database
        Required for Chado Load Tripal Synchronize workflow (which should be ran as the first workflow)
        Called outside workflow for practical reasons (Chado add doesn't have an input link for analysis or organism)

        :return:

        """

        self.connect_to_instance()
        self.set_get_history()

        tool_version = "2.3.4+galaxy0"

        get_organism_tool = self.instance.tools.show_tool("toolshed.g2.bx.psu.edu/repos/gga/chado_organism_get_organisms/organism_get_organisms/2.3.4+galaxy0")

        get_organisms = self.instance.tools.run_tool(
            tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_organism_get_organisms/organism_get_organisms/%s" % tool_version,
            history_id=self.history_id,
            tool_inputs={})

        time.sleep(10)  # Ensure the tool has had time to complete
        org_outputs = get_organisms["outputs"]  # Outputs from the get_organism tool
        org_job_out_id = org_outputs[0]["id"]  # ID of the get_organism output dataset (list of dicts)
        org_json_output = self.instance.datasets.download_dataset(dataset_id=org_job_out_id)  # Download the dataset
        org_output = json.loads(org_json_output)  # Turn the dataset into a list for parsing

        org_id = None

        # Look up list of outputs (dictionaries)
        for organism_output_dict in org_output:
            if organism_output_dict["genus"] == self.genus and organism_output_dict["species"] == "{0} {1}".format(self.species, self.sex):
                correct_organism_id = str(organism_output_dict["organism_id"])  # id needs to be a str to be recognized by chado tools
                org_id = str(correct_organism_id)


        if org_id is None:
            if self.common == "" or self.common is None:
                add_org_job = self.instance.tools.run_tool(
                    tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_organism_add_organism/organism_add_organism/%s" % tool_version,
                    history_id=self.history_id,
                    tool_inputs={"abbr": self.abbreviation,
                                 "genus": self.genus_uppercase,
                                 "species": self.chado_species_name,
                                 "common": self.abbreviation})
                org_job_out_id = add_org_job["outputs"][0]["id"]
                org_json_output = self.instance.datasets.download_dataset(dataset_id=org_job_out_id)
                org_output = json.loads(org_json_output)
                org_id = str(org_output["organism_id"])  # id needs to be a str to be recognized by chado tools
            else:
                add_org_job = self.instance.tools.run_tool(
                    tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_organism_add_organism/organism_add_organism/%s" % tool_version,
                    history_id=self.history_id,
                    tool_inputs={"abbr": self.abbreviation,
                                 "genus": self.genus_uppercase,
                                 "species": self.chado_species_name,
                                 "common": self.common})
                org_job_out_id = add_org_job["outputs"][0]["id"]
                org_json_output = self.instance.datasets.download_dataset(dataset_id=org_job_out_id)
                org_output = json.loads(org_json_output)
                org_id = str(org_output["organism_id"])  # id needs to be a str to be recognized by chado tools

        # Synchronize newly added organism in Tripal
        logging.info("Synchronizing organism %s in Tripal" % self.full_name)
        time.sleep(60)
        org_sync = self.instance.tools.run_tool(tool_id="toolshed.g2.bx.psu.edu/repos/gga/tripal_organism_sync/organism_sync/3.2.1.0",
                                                history_id=self.history_id,
                                                tool_inputs={"organism_id": org_id})


        # Analyses (genome + OGS)
        get_analyses = self.instance.tools.run_tool(
            tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_analysis_get_analyses/analysis_get_analyses/%s" % tool_version,
            history_id=self.history_id,
            tool_inputs={})

        time.sleep(10)
        analysis_outputs = get_analyses["outputs"]
        analysis_job_out_id = analysis_outputs[0]["id"]
        analysis_json_output = self.instance.datasets.download_dataset(dataset_id=analysis_job_out_id)
        analysis_output = json.loads(analysis_json_output)

        ogs_analysis_id = None
        genome_analysis_id = None

        # Look up list of outputs (dictionaries)
        for analysis_output_dict in analysis_output:
            if analysis_output_dict["name"] == self.full_name_lowercase + " OGS" + self.ogs_version:
                ogs_analysis_id = str(analysis_output_dict["analysis_id"])
            if analysis_output_dict["name"] == self.full_name_lowercase + " genome v" + self.genome_version:
                genome_analysis_id = str(analysis_output_dict["analysis_id"])


        if ogs_analysis_id is None:
            add_ogs_analysis_job = self.instance.tools.run_tool(
                tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_analysis_add_analysis/analysis_add_analysis/%s" % tool_version,
                history_id=self.history_id,
                tool_inputs={"name": self.full_name_lowercase + " OGS" + self.ogs_version,
                             "program": "Performed by Genoscope",
                             "programversion": str(self.sex + " OGS" + self.ogs_version),
                             "sourcename": "Genoscope",
                             "date_executed": self.date})
            analysis_outputs = add_ogs_analysis_job["outputs"]
            analysis_job_out_id = analysis_outputs[0]["id"]
            analysis_json_output = self.instance.datasets.download_dataset(dataset_id=analysis_job_out_id)
            analysis_output = json.loads(analysis_json_output)
            ogs_analysis_id = str(analysis_output["analysis_id"])
            
        # Synchronize OGS analysis in Tripal
        logging.info("Synchronizing OGS%s analysis in Tripal" % self.ogs_version)
        time.sleep(60)
        ogs_analysis_sync = self.instance.tools.run_tool(tool_id="toolshed.g2.bx.psu.edu/repos/gga/tripal_analysis_sync/analysis_sync/3.2.1.0",
                                                         history_id=self.history_id,
                                                         tool_inputs={"analysis_id": ogs_analysis_id})
                    
        if genome_analysis_id is None:
            add_genome_analysis_job = self.instance.tools.run_tool(
                tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_analysis_add_analysis/analysis_add_analysis/%s" % tool_version,
                history_id=self.history_id,
                tool_inputs={"name": self.full_name_lowercase + " genome v" + self.genome_version,
                             "program": "Performed by Genoscope",
                             "programversion": str(self.sex + "genome v" + self.genome_version),
                             "sourcename": "Genoscope",
                             "date_executed": self.date})
            analysis_outputs = add_genome_analysis_job["outputs"]
            analysis_job_out_id = analysis_outputs[0]["id"]
            analysis_json_output = self.instance.datasets.download_dataset(dataset_id=analysis_job_out_id)
            analysis_output = json.loads(analysis_json_output)
            genome_analysis_id = str(analysis_output["analysis_id"])
    
        # Synchronize genome analysis in Tripal
        logging.info("Synchronizing genome v%s analysis in Tripal" % self.genome_version)
        time.sleep(60)
        genome_analysis_sync = self.instance.tools.run_tool(tool_id="toolshed.g2.bx.psu.edu/repos/gga/tripal_analysis_sync/analysis_sync/3.2.1.0",
                                                            history_id=self.history_id,
                                                            tool_inputs={"analysis_id": genome_analysis_id})

        # print({"org_id": org_id, "genome_analysis_id": genome_analysis_id, "ogs_analysis_id": ogs_analysis_id})
        return({"org_id": org_id, "genome_analysis_id": genome_analysis_id, "ogs_analysis_id": ogs_analysis_id})


    def add_organism_blastp_analysis(self):
        """
        Add OGS and genome vX analyses to Chado database
        Required for Chado Load Tripal Synchronize workflow (which should be ran as the first workflow)
        Called outside workflow for practical reasons (Chado add doesn't have an input link for analysis or organism)

        :return:

        """

        self.connect_to_instance()
        self.set_get_history()

        tool_version = "2.3.4+galaxy0"

        get_organism_tool = self.instance.tools.show_tool("toolshed.g2.bx.psu.edu/repos/gga/chado_organism_get_organisms/organism_get_organisms/2.3.4+galaxy0")

        get_organisms = self.instance.tools.run_tool(
            tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_organism_get_organisms/organism_get_organisms/%s" % tool_version,
            history_id=self.history_id,
            tool_inputs={})

        time.sleep(10)  # Ensure the tool has had time to complete
        org_outputs = get_organisms["outputs"]  # Outputs from the get_organism tool
        org_job_out_id = org_outputs[0]["id"]  # ID of the get_organism output dataset (list of dicts)
        org_json_output = self.instance.datasets.download_dataset(dataset_id=org_job_out_id)  # Download the dataset
        org_output = json.loads(org_json_output)  # Turn the dataset into a list for parsing

        org_id = None

        # Look up list of outputs (dictionaries)
        for organism_output_dict in org_output:
            if organism_output_dict["genus"] == self.genus and organism_output_dict["species"] == "{0} {1}".format(self.species, self.sex):
                correct_organism_id = str(organism_output_dict["organism_id"])  # id needs to be a str to be recognized by chado tools
                org_id = str(correct_organism_id)


        if org_id is None:
            if self.common == "" or self.common is None:
                add_org_job = self.instance.tools.run_tool(
                    tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_organism_add_organism/organism_add_organism/%s" % tool_version,
                    history_id=self.history_id,
                    tool_inputs={"abbr": self.abbreviation,
                                 "genus": self.genus_uppercase,
                                 "species": self.chado_species_name,
                                 "common": self.abbreviation})
                org_job_out_id = add_org_job["outputs"][0]["id"]
                org_json_output = self.instance.datasets.download_dataset(dataset_id=org_job_out_id)
                org_output = json.loads(org_json_output)
                org_id = str(org_output["organism_id"])  # id needs to be a str to be recognized by chado tools
            else:
                add_org_job = self.instance.tools.run_tool(
                    tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_organism_add_organism/organism_add_organism/%s" % tool_version,
                    history_id=self.history_id,
                    tool_inputs={"abbr": self.abbreviation,
                                 "genus": self.genus_uppercase,
                                 "species": self.chado_species_name,
                                 "common": self.common})
                org_job_out_id = add_org_job["outputs"][0]["id"]
                org_json_output = self.instance.datasets.download_dataset(dataset_id=org_job_out_id)
                org_output = json.loads(org_json_output)
                org_id = str(org_output["organism_id"])  # id needs to be a str to be recognized by chado tools

            # Synchronize newly added organism in Tripal
            logging.info("Synchronizing organism %s in Tripal" % self.full_name)
            time.sleep(60)
            org_sync = self.instance.tools.run_tool(tool_id="toolshed.g2.bx.psu.edu/repos/gga/tripal_organism_sync/organism_sync/3.2.1.0",
                                                    history_id=self.history_id,
                                                    tool_inputs={"organism_id": org_id})


        get_analyses = self.instance.tools.run_tool(
            tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_analysis_get_analyses/analysis_get_analyses/%s" % tool_version,
            history_id=self.history_id,
            tool_inputs={})

        time.sleep(10)
        analysis_outputs = get_analyses["outputs"]
        analysis_job_out_id = analysis_outputs[0]["id"]
        analysis_json_output = self.instance.datasets.download_dataset(dataset_id=analysis_job_out_id)
        analysis_output = json.loads(analysis_json_output)

        blastp_analysis_id = None

        # Look up list of outputs (dictionaries)
        for analysis_output_dict in analysis_output:
            if analysis_output_dict["name"] == "Diamond on " + self.full_name_lowercase + " OGS" + self.ogs_version:
                blastp_analysis_id = str(analysis_output_dict["analysis_id"])


        if blastp_analysis_id is None:
            add_blast_analysis_job = self.instance.tools.run_tool(
                tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_analysis_add_analysis/analysis_add_analysis/%s" % tool_version,
                history_id=self.history_id,
                tool_inputs={"name": "Diamond on " + self.full_name_lowercase + " OGS" + self.ogs_version,
                             "program": "Performed by Genoscope",
                             "programversion": str(self.sex + " OGS" + self.ogs_version),
                             "sourcename": "Genoscope",
                             "date_executed": self.date})
            analysis_outputs = add_blast_analysis_job["outputs"]
            analysis_job_out_id = analysis_outputs[0]["id"]
            analysis_json_output = self.instance.datasets.download_dataset(dataset_id=analysis_job_out_id)
            analysis_output = json.loads(analysis_json_output)
            blastp_analysis_id = str(analysis_output["analysis_id"])

        # Synchronize blastp analysis
        logging.info("Synchronizing Diamong blastp OGS%s analysis in Tripal" % self.ogs_version)
        time.sleep(60)
        blastp_analysis_sync = self.instance.tools.run_tool(tool_id="toolshed.g2.bx.psu.edu/repos/gga/tripal_analysis_sync/analysis_sync/3.2.1.0",
                                                            history_id=self.history_id,
                                                            tool_inputs={"analysis_id": blastp_analysis_id})

        # print({"org_id": org_id, "genome_analysis_id": genome_analysis_id, "ogs_analysis_id": ogs_analysis_id})
        return({"org_id": org_id, "blastp_analysis_id": blastp_analysis_id})

    def add_organism_interproscan_analysis(self):
        """
        Add OGS and genome vX analyses to Chado database
        Required for Chado Load Tripal Synchronize workflow (which should be ran as the first workflow)
        Called outside workflow for practical reasons (Chado add doesn't have an input link for analysis or organism)

        :return:

        """

        self.connect_to_instance()
        self.set_get_history()

        tool_version = "2.3.4+galaxy0"

        get_organism_tool = self.instance.tools.show_tool("toolshed.g2.bx.psu.edu/repos/gga/chado_organism_get_organisms/organism_get_organisms/2.3.4+galaxy0")

        get_organisms = self.instance.tools.run_tool(
            tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_organism_get_organisms/organism_get_organisms/%s" % tool_version,
            history_id=self.history_id,
            tool_inputs={})

        time.sleep(10)  # Ensure the tool has had time to complete
        org_outputs = get_organisms["outputs"]  # Outputs from the get_organism tool
        org_job_out_id = org_outputs[0]["id"]  # ID of the get_organism output dataset (list of dicts)
        org_json_output = self.instance.datasets.download_dataset(dataset_id=org_job_out_id)  # Download the dataset
        org_output = json.loads(org_json_output)  # Turn the dataset into a list for parsing

        org_id = None

        # Look up list of outputs (dictionaries)
        for organism_output_dict in org_output:
            if organism_output_dict["genus"] == self.genus and organism_output_dict["species"] == "{0} {1}".format(self.species, self.sex):
                correct_organism_id = str(organism_output_dict["organism_id"])  # id needs to be a str to be recognized by chado tools
                org_id = str(correct_organism_id)


        if org_id is None:
            if self.common == "" or self.common is None:
                add_org_job = self.instance.tools.run_tool(
                    tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_organism_add_organism/organism_add_organism/%s" % tool_version,
                    history_id=self.history_id,
                    tool_inputs={"abbr": self.abbreviation,
                                 "genus": self.genus_uppercase,
                                 "species": self.chado_species_name,
                                 "common": self.abbreviation})
                org_job_out_id = add_org_job["outputs"][0]["id"]
                org_json_output = self.instance.datasets.download_dataset(dataset_id=org_job_out_id)
                org_output = json.loads(org_json_output)
                org_id = str(org_output["organism_id"])  # id needs to be a str to be recognized by chado tools
            else:
                add_org_job = self.instance.tools.run_tool(
                    tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_organism_add_organism/organism_add_organism/%s" % tool_version,
                    history_id=self.history_id,
                    tool_inputs={"abbr": self.abbreviation,
                                 "genus": self.genus_uppercase,
                                 "species": self.chado_species_name,
                                 "common": self.common})
                org_job_out_id = add_org_job["outputs"][0]["id"]
                org_json_output = self.instance.datasets.download_dataset(dataset_id=org_job_out_id)
                org_output = json.loads(org_json_output)
                org_id = str(org_output["organism_id"])  # id needs to be a str to be recognized by chado tools

            # Synchronize newly added organism in Tripal
            logging.info("Synchronizing organism %s in Tripal" % self.full_name)
            time.sleep(60)
            org_sync = self.instance.tools.run_tool(tool_id="toolshed.g2.bx.psu.edu/repos/gga/tripal_organism_sync/organism_sync/3.2.1.0",
                                                    history_id=self.history_id,
                                                    tool_inputs={"organism_id": org_id})


        get_analyses = self.instance.tools.run_tool(
            tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_analysis_get_analyses/analysis_get_analyses/%s" % tool_version,
            history_id=self.history_id,
            tool_inputs={})

        time.sleep(10)
        analysis_outputs = get_analyses["outputs"]
        analysis_job_out_id = analysis_outputs[0]["id"]
        analysis_json_output = self.instance.datasets.download_dataset(dataset_id=analysis_job_out_id)
        analysis_output = json.loads(analysis_json_output)

        interpro_analysis_id = None

        # Look up list of outputs (dictionaries)
        for analysis_output_dict in analysis_output:
            if analysis_output_dict["name"] == "Interproscan on " + self.full_name_lowercase + " OGS" + self.ogs_version:
                interpro_analysis_id = str(analysis_output_dict["analysis_id"])


        if interpro_analysis_id is None:
            add_interproscan_analysis_job = self.instance.tools.run_tool(
                tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_analysis_add_analysis/analysis_add_analysis/%s" % tool_version,
                history_id=self.history_id,
                tool_inputs={"name": "Interproscan on " + self.full_name_lowercase + " OGS" + self.ogs_version,
                             "program": "Performed by Genoscope",
                             "programversion": str(self.sex + " OGS" + self.ogs_version),
                             "sourcename": "Genoscope",
                             "date_executed": self.date})
            analysis_outputs = add_interproscan_analysis_job["outputs"]
            analysis_job_out_id = analysis_outputs[0]["id"]
            analysis_json_output = self.instance.datasets.download_dataset(dataset_id=analysis_job_out_id)
            analysis_output = json.loads(analysis_json_output)
            interpro_analysis_id = str(analysis_output["analysis_id"])

        # Synchronize blastp analysis
        logging.info("Synchronizing Diamong blastp OGS%s analysis in Tripal" % self.ogs_version)
        time.sleep(60)
        interproscan_analysis_sync = self.instance.tools.run_tool(tool_id="toolshed.g2.bx.psu.edu/repos/gga/tripal_analysis_sync/analysis_sync/3.2.1.0",
                                                            history_id=self.history_id,
                                                            tool_inputs={"analysis_id": interpro_analysis_id})

        # print({"org_id": org_id, "genome_analysis_id": genome_analysis_id, "ogs_analysis_id": ogs_analysis_id})
        return({"org_id": org_id, "interpro_analysis_id": interpro_analysis_id})


    def get_interpro_analysis_id(self):
        """
        """

        # Get interpro ID
        interpro_analysis = self.instance.tools.run_tool(
            tool_id="toolshed.g2.bx.psu.edu/repos/gga/chado_analysis_get_analyses/analysis_get_analyses/2.3.4+galaxy0",
            history_id=self.history_id,
            tool_inputs={"name": "InterproScan on OGS%s" % self.ogs_version})
        interpro_analysis_job_out = interpro_analysis["outputs"][0]["id"]
        interpro_analysis_json_output = self.instance.datasets.download_dataset(dataset_id=interpro_analysis_job_out)
        try:
            interpro_analysis_output = json.loads(interpro_analysis_json_output)[0]
            self.interpro_analysis_id = str(interpro_analysis_output["analysis_id"])
        except IndexError as exc:
            logging.critical("No matching InterproScan analysis exists in the instance's chado database")
            sys.exit(exc)

        return self.interpro_analysis_id


    def get_invocation_report(self, workflow_name):
        """
        Debugging method for workflows

        Simply logs and returns a report of the previous workflow invocation (execution of a workflow in
        the instance via the API)

        :param workflow_name:
        :return:
        """

        workflow_attributes = self.instance.workflows.get_workflows(name=workflow_name)
        workflow_id = workflow_attributes[1]["id"]  # Most recently imported workflow (index 1 in the list)
        invocations = self.instance.workflows.get_invocations(workflow_id=workflow_id)
        invocation_id = invocations[1]["id"]  # Most recent invocation
        invocation_report = self.instance.invocations.get_invocation_report(invocation_id=invocation_id)

        logging.debug(invocation_report)

        return invocation_report


    def import_datasets_into_history(self):
        """
        Find datasets in a library, get their ID and import them into the current history if they are not already

        :return:
        """

        # Instanciate the instance 
        gio = GalaxyInstance(url=self.instance_url,
                             email=self.config["galaxy_default_admin_email"],
                             password=self.config["galaxy_default_admin_password"])

        prj_lib = gio.libraries.get_previews(name="Project Data")
        library_id = prj_lib[0].id

        instance_source_data_folders = self.instance.libraries.get_folders(library_id=str(library_id))

        folders_ids = {}
        folder_name = ""

        # Loop over the folders in the library and map folders names to their IDs
        for i in instance_source_data_folders:
            folders_ids[i["name"]] = i["id"]

        # Iterating over the folders to find datasets and map datasets to their IDs
        for k, v in folders_ids.items():
            if k == "/genome/{0}/v{1}".format(self.species_folder_name, self.genome_version):
                sub_folder_content = self.instance.folders.show_folder(folder_id=v, contents=True)
                for k2, v2 in sub_folder_content.items():
                    for e in v2:
                        if type(e) == dict:
                            if e["name"].endswith(".fasta"):
                                self.datasets["genome_file"] = e["ldda_id"]
                                self.datasets_name["genome_file"] = e["name"]

            if k == "/annotation/{0}/OGS{1}".format(self.species_folder_name, self.ogs_version):
                sub_folder_content = self.instance.folders.show_folder(folder_id=v, contents=True)
                for k2, v2 in sub_folder_content.items():
                    for e in v2:
                        if type(e) == dict:
                            if "transcripts" in e["name"]:
                                self.datasets["transcripts_file"] = e["ldda_id"]
                                self.datasets_name["transcripts_file"] = e["name"]
                            elif "proteins" in e["name"]:
                                self.datasets["proteins_file"] = e["ldda_id"]
                                self.datasets_name["proteins_file"] = e["name"]
                            elif "gff" in e["name"]:
                                self.datasets["gff_file"] = e["ldda_id"]
                                self.datasets_name["gff_file"] = e["name"]
                            elif "interpro" in e["name"]:
                                self.datasets["interproscan_file"] = e["ldda_id"]
                                self.datasets_name["interproscan_file"] = e["name"]
                            elif "blastp" in e["name"]:
                                self.datasets["blastp_file"] = e["ldda_id"]
                                self.datasets_name["blastp_file"] = e["name"]


        history_datasets_li = self.instance.datasets.get_datasets()
        genome_hda_id, gff_hda_id, transcripts_hda_id, proteins_hda_id, blastp_hda_id, interproscan_hda_id = None, None, None, None, None, None

        # Finding datasets in history (matching datasets names)
        for dataset in history_datasets_li:
            dataset_name = dataset["name"]
            dataset_id = dataset["id"]
            if dataset_name == "{0}_v{1}.fasta".format(self.dataset_prefix, self.genome_version):
                genome_hda_id = dataset_id
            if dataset_name == "{0}_OGS{1}_{2}.gff".format(self.dataset_prefix, self.ogs_version, self.date):
                gff_hda_id = dataset_id
            if dataset_name == "{0}_OGS{1}_transcripts.fasta".format(self.dataset_prefix, self.ogs_version):
                transcripts_hda_id = dataset_id
            if dataset_name == "{0}_OGS{1}_proteins.fasta".format(self.dataset_prefix, self.ogs_version):
                proteins_hda_id = dataset_id
            if dataset_name == "{0}_OGS{1}_blastp.xml".format(self.dataset_prefix, self.ogs_version):
                blastp_hda_id = dataset_id
            if dataset_name == "{0}_OGS{1}_interproscan.xml".format(self.dataset_prefix, self.ogs_version):
                interproscan_hda_id = dataset_id

                    
        # Import each dataset into history if it is not imported
        logging.debug("Uploading datasets into history %s" % self.history_id)

        if genome_hda_id is None:
            genome_dataset_upload = self.instance.histories.upload_dataset_from_library(history_id=self.history_id, lib_dataset_id=self.datasets["genome_file"])
            genome_hda_id = genome_dataset_upload["id"]
        if gff_hda_id is  None:
            gff_dataset_upload = self.instance.histories.upload_dataset_from_library(history_id=self.history_id, lib_dataset_id=self.datasets["gff_file"])
            gff_hda_id = gff_dataset_upload["id"]
        if transcripts_hda_id is None:
            transcripts_dataset_upload = self.instance.histories.upload_dataset_from_library(history_id=self.history_id, lib_dataset_id=self.datasets["transcripts_file"])
            transcripts_hda_id = transcripts_dataset_upload["id"]
        if proteins_hda_id is None:
            proteins_dataset_upload = self.instance.histories.upload_dataset_from_library(history_id=self.history_id, lib_dataset_id=self.datasets["proteins_file"])
            proteins_hda_id = proteins_dataset_upload["id"]
        if interproscan_hda_id is None:
            try:
                interproscan_dataset_upload = self.instance.histories.upload_dataset_from_library(history_id=self.history_id, lib_dataset_id=self.datasets["interproscan_file"])
                interproscan_hda_id = interproscan_dataset_upload["id"]
            except Exception as exc:
                logging.debug("Interproscan file not found in library (history: {0})".format(self.history_id))
        if blastp_hda_id is None:
            try:
                blastp_dataset_upload = self.instance.histories.upload_dataset_from_library(history_id=self.history_id, lib_dataset_id=self.datasets["blastp_file"])
                blastp_hda_id = blastp_dataset_upload["id"]
            except Exception as exc:
                logging.debug("blastp file not found in library (history: {0})".format(self.history_id))

        # logging.debug("History dataset IDs (hda_id) for %s:" % self.full_name)
        # logging.debug({"genome_hda_id": genome_hda_id,
        #         "gff_hda_id": gff_hda_id,
        #         "transcripts_hda_id": transcripts_hda_id,
        #         "proteins_hda_id": proteins_hda_id,
        #         "blastp_hda_id": blastp_hda_id,
        #         "interproscan_hda_id": interproscan_hda_id})

        # Return a dict made of the hda ids
        return {"genome_hda_id": genome_hda_id, 
                "gff_hda_id": gff_hda_id, 
                "transcripts_hda_id": transcripts_hda_id, 
                "proteins_hda_id": proteins_hda_id, 
                "blastp_hda_id": blastp_hda_id,
                "interproscan_hda_id": interproscan_hda_id}


    def get_datasets_hda_ids(self):
        """
        Get the hda IDs of the datasets imported into an history

        As some tools will not work using the input datasets ldda IDs we need to retrieve the datasets IDs imported
        into an history


        :return:
        """

        # List of all datasets in the instance (including outputs from jobs)
        # "limit" and "offset" options *may* be used to restrict search to specific datasets but since
        # there is no way to know which imported datasets are the correct ones depending on history content
        # it's not currently used
        history_datasets_li = self.instance.datasets.get_datasets()

        genome_dataset_hda_id, gff_dataset_hda_id, transcripts_dataset_hda_id, proteins_datasets_hda_id = None, None, None, None
        interproscan_dataset_hda_id, blast_diamond_dataset_hda_id = None, None

        # Match files imported in history names vs library datasets names to assign their respective hda_id
        for dataset_dict in history_datasets_li:
            if dataset_dict["history_id"] == self.history_id:
                if dataset_dict["name"] == self.datasets_name["genome_file"] and dataset_dict["id"] not in imported_datasets_ids:
                    genome_dataset_hda_id = dataset_dict["id"]
                elif dataset_dict["name"] == self.datasets_name["proteins_file"] and dataset_dict["id"] not in imported_datasets_ids:
                    proteins_datasets_hda_id = dataset_dict["id"]
                elif dataset_dict["name"] == self.datasets_name["transcripts_file"] and dataset_dict["id"] not in imported_datasets_ids:
                    transcripts_dataset_hda_id = dataset_dict["id"]
                elif dataset_dict["name"] == self.datasets_name["gff_file"] and dataset_dict["id"] not in imported_datasets_ids:
                    gff_dataset_hda_id = dataset_dict["id"]
                if "interproscan_file" in self.datasets_name.keys():
                    if dataset_dict["name"] == self.datasets_name["interproscan_file"] and dataset_dict["id"] not in imported_datasets_ids:
                        interproscan_dataset_hda_id = dataset_dict["id"]
                if "blast_diamond_file" in self.datasets_name.keys():
                    if dataset_dict["name"] == self.datasets_name["blastp_file"] and dataset_dict["id"] not in imported_datasets_ids:
                        blastp_dataset_hda_id = dataset_dict["id"]
                    
        logging.debug("Genome dataset hda id: %s" % genome_dataset_hda_id)
        logging.debug("Proteins dataset hda ID: %s" % proteins_datasets_hda_id)
        logging.debug("Transcripts dataset hda ID: %s" % transcripts_dataset_hda_id)
        logging.debug("GFF dataset hda ID: %s" % gff_dataset_hda_id)
        logging.debug("InterproScan dataset hda ID: %s" % gff_dataset_hda_id)
        logging.debug("Blastp Diamond dataset hda ID: %s" % blastp_dataset_hda_id)

        # Add datasets IDs to already imported IDs (so we don't assign all the wrong IDs to the next organism if there is one)
        imported_datasets_ids.append(genome_dataset_hda_id)
        imported_datasets_ids.append(transcripts_dataset_hda_id)
        imported_datasets_ids.append(proteins_datasets_hda_id)
        imported_datasets_ids.append(gff_dataset_hda_id)
        imported_datasets_ids.append(interproscan_dataset_hda_id)
        imported_datasets_ids.append(blastp_dataset_hda_id)

        # Return a dict made of the hda ids
        return {"genome_hda_id": genome_dataset_hda_id, "transcripts_hda_id": transcripts_dataset_hda_id,
                "proteins_hda_id": proteins_datasets_hda_id, "gff_hda_id": gff_dataset_hda_id,
                "interproscan_hda_id": interproscan_dataset_hda_id,
                "blastp_hda_id": blastp_dataset_hda_id,
                "imported_datasets_ids": imported_datasets_ids}


def run_workflow(workflow_path, workflow_parameters, datamap, config, input_species_number):
    """
    Run a workflow in galaxy
    Requires the .ga file to be loaded as a dictionary (optionally could be uploaded as a raw file)

    :param workflow_name:
    :param workflow_parameters:
    :param datamap:
    :return:
    """

    logging.info("Importing workflow %s" % str(workflow_path))

    # Load the workflow file (.ga) in a buffer
    with open(workflow_path, 'r') as ga_in_file:

        # Then store the decoded json dictionary
        workflow_dict = json.load(ga_in_file)

        # In case of the Jbrowse workflow, we unfortunately have to manually edit the parameters instead of setting them
        # as runtime values, using runtime parameters makes the tool throw an internal critical error ("replace not found" error)
        # Scratchgmod test: need "http" (or "https"), the hostname (+ port)
        if "jbrowse_menu_url" not in config.keys():
            jbrowse_menu_url = "https://{hostname}/sp/{genus_sp}/feature/{Genus}/{species}/mRNA/{id}".format(hostname=self.config["hostname"], genus_sp=self.genus_species, Genus=self.genus_uppercase, species=self.species, id="{id}")
        else:
            jbrowse_menu_url = config["jbrowse_menu_url"]
        if workflow_name == "Jbrowse":
            workflow_dict["steps"]["2"]["tool_state"] = workflow_dict["steps"]["2"]["tool_state"].replace("__MENU_URL__", jbrowse_menu_url)
            # The UNIQUE_ID is specific to a combination genus_species_strain_sex so every combination should have its unique workflow
            # in galaxy --> define a naming method for these workflows
            workflow_dict["steps"]["3"]["tool_state"] = workflow_dict["steps"]["3"]["tool_state"].replace("__FULL_NAME__", self.full_name).replace("__UNIQUE_ID__", self.species_folder_name)

        # Import the workflow in galaxy as a dict
        self.instance.workflows.import_workflow_dict(workflow_dict=workflow_dict)

        # Get its attributes
        workflow_attributes = self.instance.workflows.get_workflows(name=workflow_name)
        # Then get its ID (required to invoke the workflow)
        workflow_id = workflow_attributes[0]["id"]  # Index 0 is the most recently imported workflow (the one we want)
        show_workflow = self.instance.workflows.show_workflow(workflow_id=workflow_id)
        # Check if the workflow is found
        try:
            logging.debug("Workflow ID: %s" % workflow_id)
        except bioblend.ConnectionError:
            logging.warning("Error retrieving workflow attributes for workflow %s" % workflow_name)

        # Finally, invoke the workflow alogn with its datamap, parameters and the history in which to invoke it
        self.instance.workflows.invoke_workflow(workflow_id=workflow_id,
                                                history_id=self.history_id,
                                                params=workflow_parameters,
                                                inputs=datamap,
                                                allow_tool_state_corrections=True)

        logging.info("Successfully imported and invoked workflow {0}, check the galaxy instance ({1}) for the jobs state".format(workflow_name, self.instance_url))




def create_sp_workflow_dict(sp_dict, main_dir, config, workflow_type):
    """
    """

    sp_workflow_dict = {}
    run_workflow_for_current_organism = RunWorkflow(parameters_dictionary=sp_dict)

    # Verifying the galaxy container is running
    if utilities.check_galaxy_state(genus_lowercase=run_workflow_for_current_organism.genus_lowercase,
                                    species=run_workflow_for_current_organism.species,
                                    script_dir=run_workflow_for_current_organism.script_dir):

        # Setting some of the instance attributes
        run_workflow_for_current_organism.main_dir = main_dir
        run_workflow_for_current_organism.species_dir = os.path.join(run_workflow_for_current_organism.main_dir,
                                                                     run_workflow_for_current_organism.genus_species +
                                                                     "/")

        # Parse the config yaml file
        run_workflow_for_current_organism.config = config
        # Set the instance url attribute --> TODO: the localhost rule in the docker-compose still doesn't work on scratchgmodv1
        run_workflow_for_current_organism.instance_url = "http://localhost:{0}/sp/{1}_{2}/galaxy/".format( 
            run_workflow_for_current_organism.config["http_port"],
            run_workflow_for_current_organism.genus_lowercase,
            run_workflow_for_current_organism.species)


        if workflow_type == "load_fasta_gff_jbrowse":
            run_workflow_for_current_organism.connect_to_instance()

            history_id = run_workflow_for_current_organism.set_get_history()

            run_workflow_for_current_organism.install_changesets_revisions_for_individual_tools()
            ids = run_workflow_for_current_organism.add_organism_ogs_genome_analyses()

            org_id = None
            genome_analysis_id = None
            ogs_analysis_id = None
            org_id = ids["org_id"]
            genome_analysis_id = ids["genome_analysis_id"]
            ogs_analysis_id = ids["ogs_analysis_id"]
            instance_attributes = run_workflow_for_current_organism.get_instance_attributes()
            hda_ids = run_workflow_for_current_organism.import_datasets_into_history()

            strain_sex = "{0}_{1}".format(run_workflow_for_current_organism.strain, run_workflow_for_current_organism.sex)
            genus_species = run_workflow_for_current_organism.genus_species

            # Create the dictionary holding all attributes needed to connect to the galaxy instance
            attributes = {"genus": run_workflow_for_current_organism.genus,
                          "species": run_workflow_for_current_organism.species,
                          "genus_species": run_workflow_for_current_organism.genus_species,
                          "full_name": run_workflow_for_current_organism.full_name,
                          "species_folder_name": run_workflow_for_current_organism.species_folder_name,
                          "sex": run_workflow_for_current_organism.sex,
                          "strain": run_workflow_for_current_organism.strain,
                          "org_id": org_id,
                          "genome_analysis_id": genome_analysis_id,
                          "ogs_analysis_id": ogs_analysis_id,
                          "instance_attributes": instance_attributes,
                          "hda_ids": hda_ids,
                          "history_id": history_id,
                          "instance": run_workflow_for_current_organism.instance,
                          "instance_url": run_workflow_for_current_organism.instance_url,
                          "email": config["galaxy_default_admin_email"],
                          "password": config["galaxy_default_admin_password"]}

            sp_workflow_dict[genus_species] = {strain_sex: attributes}

        else:
            logging.critical("The galaxy container for %s is not ready yet!" % run_workflow_for_current_organism.full_name)
            sys.exit()

        return sp_workflow_dict

    if workflow_type == "blast":
        run_workflow_for_current_organism.connect_to_instance()

        history_id = run_workflow_for_current_organism.set_get_history()

        run_workflow_for_current_organism.install_changesets_revisions_for_individual_tools()
        ids = run_workflow_for_current_organism.add_organism_blastp_analysis()

        org_id = None
        org_id = ids["org_id"]
        blastp_analysis_id = None
        blastp_analysis_id = ids["blastp_analysis_id"]
        instance_attributes = run_workflow_for_current_organism.get_instance_attributes()
        hda_ids = run_workflow_for_current_organism.import_datasets_into_history()

        strain_sex = "{0}_{1}".format(run_workflow_for_current_organism.strain, run_workflow_for_current_organism.sex)
        genus_species = run_workflow_for_current_organism.genus_species

        # Create the dictionary holding all attributes needed to connect to the galaxy instance
        attributes = {"genus": run_workflow_for_current_organism.genus,
                      "species": run_workflow_for_current_organism.species,
                      "genus_species": run_workflow_for_current_organism.genus_species,
                      "full_name": run_workflow_for_current_organism.full_name,
                      "species_folder_name": run_workflow_for_current_organism.species_folder_name,
                      "sex": run_workflow_for_current_organism.sex,
                      "strain": run_workflow_for_current_organism.strain,
                      "org_id": org_id,
                      "blastp_analysis_id": blastp_analysis_id,