gga_init.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import argparse
import os
import re
import subprocess
import logging
import sys
import yaml
import shutil

from pathlib import Path
from jinja2 import Template, Environment, FileSystemLoader

import utilities
import speciesData

""" 
gga_init.py

Usage: $ python3 gga_init.py -i input_example.yml --config config.yml [OPTIONS]


TODO
- Exclude traefik dir tree creation and file writing from the loop (make make_dirs() an external func and write a func similar to make_compose_files()
  for traefik and authelia as an external func)

"""

class DeploySpeciesStack(speciesData.SpeciesData):
    """
    Child of SpeciesData

    Contains methods and attributes to deploy a stack of services for a given organism, from creating/updating
    the organism's directory tree to create the required docker-compose files and stack deployment

    """

    def make_directory_tree(self):
        """
        Generate the directory tree for an organism

        :return:
        """

        os.chdir(self.main_dir)

        # Create the species main directory (name of the dir: genus_species)
        try:
            os.mkdir(self.species_dir)
        except FileExistsError:
            logging.info("Updating directory tree of %s" % self.genus_species)
        try:
            os.chdir(self.species_dir)
        except OSError as exc:
            logging.critical("Cannot access %s" % self.genus_species)
            sys.exit(exc)

        # Copy the custom banner to the species dir (banner used in tripal pages)
        # If the path specified is invalid (because it's empty or is still the default demo one),
        # use the default banner instead
        if "banner_path" in self.config.keys():
            if self.config["banner_path"] != "/path/to/banner" or self.config["banner_path"] != "":
                try:
                    logging.debug("Custom banner path: %s" % self.config["banner_path"])
                    if os.path.isfile(os.path.abspath(self.config["banner_path"])):
                        shutil.copy(os.path.abspath(self.config["banner_path"]), "%s/banner.png" % self.species_dir)
                except FileNotFoundError:
                    logging.warning("Specified banner not found (%s), using default banner instead" % self.config["banner_path"])
                    self.config.pop("banner_path", None)
            else:
                logging.debug("Using default banner for Tripal pages")
                self.config.pop("banner_path", None)
        else:
            logging.debug("Using default banner for Tripal pages")
            self.config.pop("banner_path", None)

        # Create nginx dirs and write/re-write nginx conf
        make_dirs(dir_paths_li=["./nginx", "./nginx/conf"])
        try:
            with open(os.path.abspath("./nginx/conf/default.conf"), 'w') as conf:
                conf.write("server {\n\tlisten 80;\n\tserver_name ~.;\n\tlocation /download/ {\n\t\talias /project_data/; \n\t\tautoindex on;\n\t}\n}")  # The species nginx conf
        except OSError as exc:
            logging.critical("Cannot edit NginX conf file")
            sys.exit(exc)

        # Creation (or updating) of the src_data directory tree
        try:
            os.mkdir("./src_data")
        except FileExistsError:
            logging.debug("'src_data' directory already exist for %s" % self.full_name)
        except PermissionError as exc:
            logging.critical("Insufficient permission to create src_data directory tree")
            sys.exit(exc)

        # List of all the directories to create in src_data
        src_data_dirs_li = ["./src_data", "./src_data/annotation", "./src_data/genome", "./src_data/tracks",
                            "./src_data/annotation/%s" % self.species_folder_name,
                            "./src_data/genome/%s" % self.species_folder_name,
                            "./src_data/annotation/{0}/OGS{1}/".format(self.species_folder_name, self.ogs_version),
                            "./src_data/genome/{0}/v{1}".format(self.species_folder_name, self.genome_version)]
        make_dirs(dir_paths_li=src_data_dirs_li)

        # Return to main directory
        os.chdir(self.main_dir)

        logging.info("Directory tree generated for %s" % self.full_name)


    def make_compose_files(self):
        """
        Create a formatted copy of the template compose file inside a species directory tree

        :return:
        """

        os.chdir(self.main_dir)
        try:
            os.chdir(self.species_dir)
        except OSError as exc:
            logging.critical("Cannot access %s" % self.species_dir)
            sys.exit(exc)

        # Jinja2 templating, handled using the python "jinja2" module
        file_loader = FileSystemLoader(self.script_dir + "/templates")
        env = Environment(loader=file_loader)

        # We need a dict holding all key (variables) - values that needs to be replaced in the template as our rendering dict
        # To do so we need both input file vars and config vars
        # Create input file vars dict
        input_vars = {"genus": self.genus_lowercase, "Genus": self.genus_uppercase, "species": self.species,
                      "genus_species": self.genus_species, "genus_species_strain_sex": self.full_name,
                      "strain": self.strain, "sex": self.sex, "Genus_species": self.genus_species[0].upper() + self.genus_species[1:]}
        # Merge the two dicts
        render_vars = {**self.config, **input_vars}

        # Render the gspecies docker-compose file and write it
        gspecies_compose_template = env.get_template("gspecies_compose_template.yml.j2")
        gspecies_compose_output = gspecies_compose_template.render(render_vars)
        with open(os.path.join(self.species_dir, "docker-compose.yml"), "w") as gspecies_compose_file:
            logging.info("Writing %s docker-compose.yml" % self.genus_species)
            gspecies_compose_file.truncate(0)
            gspecies_compose_file.write(gspecies_compose_output)

        # Create the volumes (directory) of the species docker-compose file
        create_mounts(working_dir=".", main_dir=self.main_dir)

        # Return to main directory
        os.chdir(self.main_dir)


    def make_orthology_compose_files(self):
        """

        :return:
        """

        os.chdir(self.main_dir)

        make_dirs["./orthology", "./orthology/src_data", "./orthology/src_data/genomes", 
                       "./orthology/src_data/gff", "./orthology/src_data/newicks", "./orthology/src_data/proteomes"]


def make_dirs(dir_paths_li):
    """
    Recursively create directories from a list of paths with a try-catch condition

    :param dir_paths_li:
    :return:
    """
    created_dir_paths_li = []

    for dir_path in dir_paths_li:
        try:
            os.mkdir(dir_path)
        except FileExistsError:
            logging.debug("%s directory already exists" % dir_path)
        except PermissionError as exc:
            logging.critical("Insufficient permission to create %s" % dir_path)
            sys.exit(exc)
        created_dir_paths_li.append(dir_path)

    return created_dir_paths_li

def make_traefik_compose_files(config, main_dir):
        """
        Create or update the traefik directory, docker-compose file and authelia conf files
        Only called when the argument "--traefik" is specified
        
        :param config:
        :param main_dir:
        :return:
        """

        script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
        render_vars = config

        os.chdir(main_dir)

        # Create directory tree
        make_dirs(["./traefik", "./traefik/authelia"])

        # Render and try to write the traefik docker-compose file
        # This new docker-compose file will not overwrite the one already present in the traefik dir
        # unless the argument "--overwrite-all" is specified

        # Jinja2 templating, handled using the python "jinja2" module
        file_loader = FileSystemLoader(script_dir + "/templates")
        env = Environment(loader=file_loader)

        if not os.path.isfile("./traefik/docker-compose.yml"):
            traefik_compose_template = env.get_template("traefik_compose_template.yml.j2")
            traefik_compose_output = traefik_compose_template.render(render_vars)
            with open(os.path.join(main_dir, "traefik/docker-compose.yml"), 'w') as traefik_compose_file:
                logging.info("Writing traefik docker-compose.yml")
                traefik_compose_file.truncate(0)
                traefik_compose_file.write(traefik_compose_output)

        if config["authelia_config_path"]:
            if not config["authelia_config_path"] == "" or not config["authelia_config_path"] == "/path/to/authelia/config":
                if os.path.isfile(os.path.abspath(config["authelia_config_path"])):
                    try:
                        shutil.copy(os.path.abspath(config["authelia_config_path"]), "./traefik/authelia")
                    except Exception as exc:
                        logging.critical("Cannot copy custom Authelia config file (%s)" % config["authelia_config_path"])
                        sys.exit(exc)
                else:
                    logging.critical("Custom Authelia config file not found (%s)" % config["authelia_config_path"])

        # Path to the authelia users in the repo
        authelia_users_path = script_dir + "/templates/authelia_users_template.yml"
        # Copy authelia "users" file
        if not os.path.isfile("./traefik/authelia/users.yml"):
            shutil.copy(authelia_users_path, "./traefik/authelia/users.yml")

        # Create the mounts for the traefik and authelia services
        traefik_dir = os.path.abspath(os.path.join(main_dir, "traefik"))
        if not os.path.isdir(os.path.join(traefik_dir, "docker_data")):
            create_mounts(working_dir=traefik_dir, main_dir=main_dir)

        # Return to main directory
        os.chdir(main_dir)

def create_mounts(working_dir, main_dir):
        """
        Create the folders (volumes) required by a container (to see required volumes, check their compose file)

        :return:
        """

        # Change directory to create mount points for the container
        try:
            os.chdir(os.path.abspath(working_dir))
        except Exception as exc:
            logging.critical("Cannot access %s, exiting" % working_dir)
            sys.exit(exc)
        compose_yml = os.path.abspath("./docker-compose.yml")
        if not os.path.isfile(compose_yml):
            raise Exception("Could not find docker-compose.yml at %s" % compose_yml)

        with open(compose_yml) as f:
            compose = yaml.safe_load(f)

        if 'services' not in compose:
            raise Exception("Could not find services tag in docker-compose.yml")

        # Iterate over all services to find the "volumes" we need to create
        for service in compose['services']:
            if 'volumes' in compose['services'][service]:
                for volume in compose['services'][service]['volumes']:
                    # regex to match the volumes of the services
                    reg = re.match(r"^(\./[^:]+/):[^:]+(:\w+)?$", volume)
                    if reg:
                        vol_dir = os.path.abspath('./' + reg.group(1))
                        if not os.path.exists(vol_dir):
                            os.makedirs(vol_dir, exist_ok=True)
                    else:
                        reg = re.match(r"^(\./[^:]+):[^:]+(:\w+)?$", volume)
                        if reg:
                            vol_file = os.path.abspath('./' + reg.group(1))
                            vol_dir = os.path.dirname(vol_file)
                            if not os.path.exists(vol_dir):
                                os.makedirs(vol_dir, exist_ok=True)
                            if not os.path.exists(vol_file):
                                Path(vol_file).touch()

        # Go back to the "main" directory
        try:
            os.chdir(os.path.abspath(main_dir))
        except OSError as exc:
            logging.critical("Cannot access %s, exiting" % main_dir)
            sys.exit(exc)

def deploy_stacks(input_list, main_dir):
    """
    This function first deploys/redeploys the traefik stack, then deploys/redeploys the organism stack, then redeploys the traefik stack
    This function is executed outside the "main" loop of input species

    :return:
    """

    main_dir = os.path.abspath(main_dir)
    os.chdir(main_dir)

    # Get species for which to deploy the stacks
    to_deploy_species_li = utilities.get_species_to_deploy(sp_dict_list=input_list)

    # Create the swarm cluster if needed
    subprocess.call(["docker", "swarm", "init"],
                    stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=main_dir)
    
    # Deploy traefik stack
    os.chdir("./traefik")
    subprocess.call(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "traefik"],
                                            stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=".")
    os.chdir(main_dir)

    # Deploy individual species stacks
    for sp in to_deploy_species_li:
        os.chdir(sp)
        subprocess.call(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "{0}_{1}".format(sp.split("_")[0], sp.split("_")[1])],
                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=".")
        logging.info("Deployed %s stack" % sp)
        os.chdir(main_dir)

    # Update traefik stack
    os.chdir("./traefik")
    subprocess.call(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "traefik"],
                                            stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=".")
    os.chdir(main_dir)

    # # Using deploy.sh script (obsolete)
    # Launch and update docker stacks
    # noinspection PyArgumentList
    # deploy_stacks_popen = subprocess.Popen(["sh", self.script_dir + "/deploy.sh", self.genus_species,
    #                                         self.main_dir + "/traefik"],
    #                                        stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
    #                                        universal_newlines=True)
    #
    # for stdout_line in iter(deploy_stacks_popen.stdout.readline, ""):
    #     if "daemon" in stdout_line:  # Ignore swarm init error output
    #         pass
    #     else:
    #         logging.info("\t%s" % stdout_line.strip())
    # deploy_stacks_popen.stdout.close()


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Automatic data loading in containers and interaction "
                                                 "with galaxy instances for GGA"
                                                 ", following the protocol @ "
                                                 "http://gitlab.sb-roscoff.fr/abims/e-infra/gga")

    parser.add_argument("input",
                        type=str,
                        help="Input file (yml)")

    parser.add_argument("-v", "--verbose",
                        help="Increase output verbosity",
                        action="store_true")

    parser.add_argument("--config",
                        type=str,
                        help="Config path, default to the 'config' file inside the script repository")

    parser.add_argument("--main-directory",
                        type=str,
                        help="Where the stack containers will be located, defaults to current directory")

    parser.add_argument("--traefik",
                        help="Initialize/wverwrite traefik directory all docker-compose and conf files in the traefik and authelia directories (default=False)",
                        action="store_true")

    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    # Parsing the config file if provided, using the default config otherwise
    if not args.config:
        args.config = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "config")
    else:
        args.config = os.path.abspath(args.config)

    main_dir = None
    if not args.main_directory:
        main_dir = os.getcwd()
    else:
        main_dir = os.path.abspath(args.main_directory)

    sp_dict_list = utilities.parse_input(os.path.abspath(args.input))

    # Create traefik directory and compose files if specified
    if args.traefik:
        config = utilities.parse_config(args.config)
        make_traefik_compose_files(config=config, main_dir=main_dir)

    logging.info("Deploying stacks for organisms in input file %s" % args.input)
    for sp_dict in sp_dict_list:

        # Init instance
        deploy_stack_for_current_organism = DeploySpeciesStack(parameters_dictionary=sp_dict)

        # Setting some of the instance attributes
        deploy_stack_for_current_organism.main_dir = main_dir
        deploy_stack_for_current_organism.species_dir = os.path.join(deploy_stack_for_current_organism.main_dir,
                                                                     deploy_stack_for_current_organism.genus_species +
                                                                     "/")

        # Parse the config yaml file
        deploy_stack_for_current_organism.config = utilities.parse_config(args.config)

        # Set the instance url attribute
        for env_variable, value in deploy_stack_for_current_organism.config.items():
            if env_variable == "hostname":
                deploy_stack_for_current_organism.instance_url = value + \
                                                                 deploy_stack_for_current_organism.genus_lowercase + \
                                                                 "_" + deploy_stack_for_current_organism.species + \
                                                                 "/galaxy/"
                break
            else:
                deploy_stack_for_current_organism.instance_url = "http://localhost:8888/sp/{0}_{1}/galaxy/".format(
                    deploy_stack_for_current_organism.genus_lowercase,
                    deploy_stack_for_current_organism.species)
                                                                    
        # Starting
        logging.info("gga_init.py called for %s" % deploy_stack_for_current_organism.full_name)

        # Make/update directory tree
        deploy_stack_for_current_organism.make_directory_tree()
        logging.info("Successfully generated the directory tree for %s" % deploy_stack_for_current_organism.full_name)

        # Make compose files
        deploy_stack_for_current_organism.make_compose_files()
        logging.info("Successfully generated the docker-compose files for %s" % deploy_stack_for_current_organism.full_name)

    logging.info("Deploying stacks")
    deploy_stacks(input_list=sp_dict_list, main_dir=main_dir)
    logging.info("All stacks deployed for organisms in input file %s" % args.input)