gga_init.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

import argparse
import os
import subprocess
import logging
import sys
import utilities
import shutil
import speciesData


""" 
gga_init.py

Usage: $ python3 gga_init.py -i example.yml [OPTIONS]

"""


class DeploySpeciesStack(speciesData.SpeciesData):
    """
    Child of SpeciesData

    Contains methods and attributes to deploy a stack of services for a given organism, from creating/updating
    the organism's directory tree to create the required docker-compose files
    """


    def make_directory_tree(self):
        """
        Generate the directory tree for an organism and move datasets into src_data

        :return:
        """

        os.chdir(self.main_dir)

        try:
            os.mkdir(self.species_dir)
            logging.info("Making directory tree for %s" % self.full_name)
        except FileExistsError:
            logging.info("Updating directory tree for %s" % self.full_name)
        try:
            os.chdir(self.species_dir)
            working_dir = os.getcwd()
        except OSError:
            logging.critical("Cannot access " + self.species_dir + ", run with higher privileges")
            sys.exit()

        try:
            os.mkdir("./nginx/")
            os.mkdir("./nginx/conf")
            with open(os.path.abspath("./nginx/conf/default.conf"), 'w') as conf:
                conf.write("server {\n\tlisten 80;\n\tserver_name ~.;\n\tlocation /download/ {\n\t\talias /project_data/; \n\t\tautoindex on;\n\t}\n}")  # The species nginx conf
        except FileExistsError:
            logging.debug("NginX conf exists")


        organism_annotation_dir, organism_genome_dir = None, None

        # Creation (or updating) of the src_data directory tree
        # Depth 0
        try:
            os.mkdir("./src_data")
        except FileExistsError:
            logging.debug("src_data folder already exist for %s" % self.full_name)
        except PermissionError:
            logging.critical("Insufficient permission to create src_data directory tree")
            sys.exit()

        # Depth 1
        try:
            os.mkdir("./src_data/annotation")
            os.mkdir("./src_data/genome")
            os.mkdir("./src_data/tracks")
        except FileExistsError:
            logging.debug("Depth 1 src_data folder(s) already exist for %s" % self.full_name)
        except PermissionError:
            logging.critical("Insufficient permission to create src_data directory tree")
            sys.exit()

        # Depth 2
        try:
            os.mkdir("./src_data/annotation/" + self.species_folder_name)
            os.mkdir("./src_data/genome/" + self.species_folder_name)
        except FileExistsError:
            logging.debug("Depth 2 src_data folder(s) already exist for %s" % self.full_name)
        except PermissionError:
            logging.critical("Insufficient permission to create src_data directory tree")
            sys.exit()

        # Depth 3
        try:
            os.mkdir("./src_data/annotation/" + self.species_folder_name + "/OGS" + self.ogs_version)
            os.mkdir("./src_data/genome/" + self.species_folder_name + "/v" + self.genome_version)
            organism_annotation_dir = os.path.abspath("./src_data/annotation/" + self.species_folder_name + "/OGS" + self.genome_version)
            organism_genome_dir = os.path.abspath("./src_data/genome/" + self.species_folder_name + "/v" + self.genome_version)
        except FileExistsError:
            logging.debug("Depth 3 src_data folder(s) already exist for %s" % self.full_name)
        except PermissionError:
            logging.critical("Insufficient permission to create src_data directory tree")
            sys.exit()

        # Return to main_dir
        os.chdir(self.main_dir)


    def make_compose_files(self):
        """
        Create a formatted copy of the template compose file inside a species directory tree

        :return:
        """

        os.chdir(self.main_dir)
        try:
            os.chdir(self.species_dir)
        except OSError:
            logging.critical("Cannot access " + self.species_dir)
            sys.exit(0)

        # Path to the templates used to generate the custom docker-compose files for an input species
        stack_template_path = self.script_dir + "/templates/compose_template.yml"
        traefik_template_path = self.script_dir + "/templates/traefik.yml"
        authelia_config_path = self.script_dir + "/templates/authelia_config.yml"
        authelia_users_path = self.script_dir + "/templates/authelia_users.yml"

        if self.sex and self.strain:
            genus_species_strain_sex = "_".join([self.genus.lower(), self.species, self.strain, self.sex])
        else:
            genus_species_strain_sex = "{0}_{1}".format(self.genus.lower(), self.species)

        with open(stack_template_path, 'r') as infile:
            organism_content = list()
            for line in infile:
                # Replace placeholders in the compose file and append line to output
                organism_content.append(
                    line.replace("genus_species",
                                 str(self.genus.lower() + "_" + self.species)).replace("Genus species",
                                 str(self.genus_uppercase + " " + self.species)).replace("Genus/species",
                                 str(self.genus_uppercase + "/" + self.species)).replace("gspecies",
                                 str(self.genus.lower()[0] + self.species)).replace("genus_species_strain_sex",
                                 genus_species_strain_sex))

            # Write output compose file
            with open("./docker-compose.yml", 'w') as outfile:
                outfile.truncate(0)  # Delete file content
                for line in organism_content:  # Replace env variables by those in the config file
                    for env_variable, value in self.config.items():
                        if env_variable in line:
                            line = line.replace(env_variable, value)
                            break
                    outfile.write(line)

            # Call create_mounts.py (replace subprocess.DEVNULL by subprocess.PIPE to get script stdout and stderr back)
            subprocess.call(["python3", self.script_dir + "/create_mounts.py"], cwd=self.species_dir,
                            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)  # Create mounts for the containers

        try:
            shutil.copy(os.path.join(self.script_dir, "galaxy_data_libs_SI.py"),  "../galaxy_data_libs_SI.py")
        except FileExistsError:
            logging.debug("galxy_data_libs_SI.py already exists")

        try:
            os.mkdir("../traefik")
            os.mkdir("../traefik/authelia")
            shutil.copy(authelia_config_path, "../traefik/authelia/configuration.yml")
            shutil.copy(authelia_users_path, "../traefik/authelia/users.yml")
            subprocess.call(["python3", self.script_dir + "/create_mounts.py"], cwd=self.species_dir,
                            stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)  # Create mounts for the containers
        except FileExistsError:
            logging.debug("Traefik directory already exists: %s" % os.path.abspath("../traefik"))
        try:
            shutil.copy(traefik_template_path, "../traefik/docker-compose.yml")
        except FileExistsError:
            logging.debug("Traefik compose file already exists: %s" % os.path.abspath("../traefik/docker-compose.yml"))
        subprocess.call(["python3", self.script_dir + "/create_mounts.py"], cwd=self.species_dir)

        os.chdir(self.main_dir)


    def deploy_stack(self):
        """
        Call the script "deploy.sh" used to initiliaze the swarm cluster if needed and
        launch/update the current organism's stack

        This script first try to deploy the traefik stack, then deploy the organism stack, then update the traefik stack
        The stacks are updated if already deployed

        :return:
        """

        # Launch and update docker stacks
        # noinspection PyArgumentList
        deploy_stacks_popen = subprocess.Popen(["sh", self.script_dir + "/deploy.sh", self.genus_species,
                                                self.main_dir + "/traefik"],
                                               stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                                               universal_newlines=True)

        for stdout_line in iter(deploy_stacks_popen.stdout.readline, ""):
            if "daemon" in stdout_line:  # Ignore swarm init error output
                pass
            else:
                logging.info("\t%s" % stdout_line.strip())
        deploy_stacks_popen.stdout.close()


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Automatic data loading in containers and interaction "
                                                 "with galaxy instances for GGA"
                                                 ", following the protocol @ "
                                                 "http://gitlab.sb-roscoff.fr/abims/e-infra/gga")

    parser.add_argument("input",
                        type=str,
                        help="Input file (yml)")

    parser.add_argument("-v", "--verbose",
                        help="Increase output verbosity",
                        action="store_true")

    parser.add_argument("--config",
                        type=str,
                        help="Config path, default to the 'config' file inside the script repository")

    parser.add_argument("--main-directory",
                        type=str,
                        help="Where the stack containers will be located, defaults to working directory")

    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    # Parsing the config file if provided, using the default config otherwise
    if not args.config:
        args.config = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "config")
    else:
        args.config = os.path.abspath(args.config)

    if not args.main_directory:
        args.main_directory = os.getcwd()
    else:
        args.main_directory = os.path.abspath(args.main_directory)

    sp_dict_list = utilities.parse_input(os.path.abspath(args.input))

    logging.info("Deploying stacks for organisms in input file %s" % args.input)
    for sp_dict in sp_dict_list:

        # Init instance
        deploy_stack_for_current_organism = DeploySpeciesStack(parameters_dictionary=sp_dict)

        # Setting some of the instance attributes
        deploy_stack_for_current_organism.main_dir = os.getcwd()
        deploy_stack_for_current_organism.species_dir = os.path.join(deploy_stack_for_current_organism.main_dir,
                                                                     deploy_stack_for_current_organism.genus_species +
                                                                     "/")
        deploy_stack_for_current_organism.config = utilities.parse_config(args.config)

        # Starting
        logging.info("gga_init.py called for %s" % deploy_stack_for_current_organism.full_name)

        # Make/update directory tree
        deploy_stack_for_current_organism.make_directory_tree()
        logging.info("Successfully generated the directory tree for %s" % deploy_stack_for_current_organism.full_name)

        # Make compose files
        deploy_stack_for_current_organism.make_compose_files()
        logging.info("Successfully generated the docker-compose files for %s" % deploy_stack_for_current_organism.full_name)

        # Deploy the stack
        logging.info("Deploying stack for %s..." % deploy_stack_for_current_organism.full_name)
        deploy_stack_for_current_organism.deploy_stack()
        logging.info("Successfully deployed stack for %s" % deploy_stack_for_current_organism.full_name)

        logging.info("Stack deployed for %s" % deploy_stack_for_current_organism.full_name)

    logging.info("All stacks deployed for organisms in input file %s" % args.input)