gga_init.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import argparse
import os
import re
import subprocess
import logging
import sys
import yaml
import shutil
from pathlib import Path
from jinja2 import Environment, FileSystemLoader

import utilities
import speciesData
import constants


""" 
gga_init.py

Usage: $ python3 gga_init.py -i input_example.yml --config config.yml [OPTIONS]


TODO
- Exclude traefik dir tree creation and file writing from the loop (make make_dirs() an external func and write a func similar to make_compose_files()
  for traefik and authelia as an external func)

"""

class DeploySpeciesStack(speciesData.SpeciesData):
    """
    Child of SpeciesData

    Contains methods and attributes to deploy a stack of services for a given organism, from creating/updating
    the organism's directory tree to create the required docker-compose files and stack deployment

    """

    def make_directory_tree(self):
        """
        Generate the directory tree for an organism

        :return:
        """

        os.chdir(self.main_dir)

        # Create the species main directory (name of the dir: genus_species)
        try:
            os.mkdir(self.species_dir)
        except FileExistsError:
            logging.info("Updating directory tree of %s" % self.genus_species)
        try:
            os.chdir(self.species_dir)
        except OSError as exc:
            logging.critical("Cannot access %s" % self.genus_species)
            sys.exit(exc)

        # Copy the custom banner to the species dir (banner used in tripal pages)
        # If the path specified is invalid (because it's empty or is still the default demo one),
        # use the default banner instead
        if constants.CONF_TRIPAL_BANNER_PATH in self.config.keys():
            if not config[constants.CONF_TRIPAL_BANNER_PATH] == "" and os.path.isfile(os.path.abspath(config[constants.CONF_TRIPAL_BANNER_PATH])):
                banner_dest_path = os.path.join(self.species_dir, os.path.abspath("banner.png"))
                if not os.path.isfile(banner_dest_path) and not os.path.islink(banner_dest_path):
                    os.symlink(os.path.abspath(self.config[constants.CONF_TRIPAL_BANNER_PATH]), banner_dest_path)
                    logging.info("Custom banner added: symlink from %s" % self.config[constants.CONF_TRIPAL_BANNER_PATH])
                else:
                    logging.info("Banner already exists at %s. The banner defined in config is not added." % banner_dest_path)
            else:
                logging.debug("Using default banner for Tripal pages because %s is not valid in 'config' file" % constants.CONF_TRIPAL_BANNER_PATH)
                self.config.pop(constants.CONF_TRIPAL_BANNER_PATH, None)
        else:
            logging.debug("Using default banner for Tripal pages")
            self.config.pop(constants.CONF_TRIPAL_BANNER_PATH, None)

        # Copy the organism picture for tripal
        if self.picture_path is not None:
            if os.path.isfile(self.picture_path):
                picture_path_basename = os.path.basename(self.picture_path)
                picture_path_filename, picture_path_extension = os.path.splitext(picture_path_basename)
                if picture_path_extension == ".png" or picture_path_extension == ".jpg":
                    picture_dest_name = "species%s" % picture_path_extension
                    picture_dest_path = os.path.join(self.species_dir, picture_dest_name)
                    shutil.copy(self.picture_path, picture_dest_path)
                else:
                    logging.error("Specified organism picture has wrong extension (must be '.png' or '.jpg'): {0}".format(self.picture_path))
            else:
                logging.error("Specified organism picture not found {0} for {1}".format(self.picture_path, self.genus_uppercase + " " + self.species))
                
        # Create nginx dirs and write/re-write nginx conf
        make_dirs(dir_paths_li=["./nginx", "./nginx/conf"])
        try:
            shutil.copy(os.path.join(self.script_dir, "files/nginx_download.conf"), os.path.abspath("./nginx/conf/default.conf"))
        except Exception as exc:
            logging.critical("Could not copy nginx configuration file for %s %s", self.genus, self.species)
            logging.critical(exc)            

        # Return to main directory
        os.chdir(self.main_dir)

        logging.info("Directory tree generated for %s %s", self.genus, self.species)


    def make_compose_files(self):
        """
        Create a formatted copy of the template compose file inside a species directory tree

        :return:
        """

        os.chdir(self.main_dir)
        try:
            os.chdir(self.species_dir)
        except OSError as exc:
            logging.critical("Cannot access %s" % self.species_dir)
            sys.exit(exc)

        # Jinja2 templating, handled using the python "jinja2" module
        file_loader = FileSystemLoader(self.script_dir + "/templates")
        env = Environment(loader=file_loader, trim_blocks=True, lstrip_blocks=True)

        # We need a dict holding all key (variables) - values that needs to be replaced in the template as our rendering dict
        # To do so we need both input file vars and config vars
        # Create input file vars dict

        input_vars = {"genus": self.genus_lowercase, "Genus": self.genus_uppercase, "species": self.species,
                        "genus_species": self.genus_species, "genus_species_strain_sex": self.species_folder_name,
                        "genus_species_sex": "{0}_{1}_{2}".format(self.genus_lowercase, self.species_lowercase, self.sex),
                        "strain": self.strain, "sex": self.sex, "Genus_species": "{0} {1}".format(self.genus_uppercase, self.species_lowercase),
                        "blast": self.blast, "go": self.go, "picture_path": self.picture_path}
        if (len(self.config.keys()) == 0):
            logging.error("Empty config dictionary")
        # Merge the two dicts
        render_vars = {**self.config, **input_vars}

        # Render the gspecies docker-compose file and write it
        gspecies_compose_template = env.get_template("gspecies_compose.yml.j2")
        gspecies_compose_output = gspecies_compose_template.render(render_vars)
        with open(os.path.join(self.species_dir, "docker-compose.yml"), "w") as gspecies_compose_file:
            logging.info("Writing %s docker-compose.yml" % self.genus_species)
            gspecies_compose_file.truncate(0)
            gspecies_compose_file.write(gspecies_compose_output)

        if not os.path.isfile(os.path.join(self.main_dir, "galaxy_nginx.conf")):
            galaxy_nginx_conf_template = env.get_template("galaxy_nginx.conf.j2")
            galaxy_nginx_conf_output = galaxy_nginx_conf_template.render(render_vars)
            with open(os.path.join(self.main_dir, "galaxy_nginx.conf"), "w") as galaxy_nginx_conf_file:
                logging.debug("Writing the galaxy_nginx.conf file for %s" % self.genus_species)
                galaxy_nginx_conf_file.truncate(0)
                galaxy_nginx_conf_file.write(galaxy_nginx_conf_output)
        else:
            logging.debug("galaxy_nginx.conf already exists")

        # Create the volumes (directory) of the species docker-compose file
        create_mounts(working_dir=".", main_dir=self.main_dir)

        # Return to main directory
        os.chdir(self.main_dir)


    def make_orthology_compose_files(self):
        """
        Create/update orthology compose files

        :return:
        """

        os.chdir(self.main_dir)

        make_dirs["./orthology", "./orthology/src_data", "./orthology/src_data/genomes", 
                       "./orthology/src_data/gff", "./orthology/src_data/newicks", "./orthology/src_data/proteomes"]


def make_dirs(dir_paths_li):
    """
    Recursively create directories from a list of paths with a try-catch condition

    :param dir_paths_li:
    :return:
    """
    created_dir_paths_li = []

    for dir_path in dir_paths_li:
        try:
            os.mkdir(dir_path)
        except FileExistsError:
            logging.debug("%s directory already exists" % dir_path)
        except PermissionError as exc:
            logging.critical("Insufficient permission to create %s" % dir_path)
            sys.exit(exc)
        created_dir_paths_li.append(dir_path)

    return created_dir_paths_li

def make_traefik_compose_files(config, main_dir):
        """
        Create or update the traefik directory, docker-compose file and authelia conf files
        Only called when the argument "--traefik" is specified
        
        :param config:
        :param main_dir:
        :return:
        """

        script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
        render_vars = config

        os.chdir(main_dir)

        # Create directory tree
        make_dirs(["./traefik", "./traefik/authelia"])

        # Render and try to write the traefik docker-compose file
        # This new docker-compose file will not overwrite the one already present in the traefik dir
        # unless the argument "--overwrite-all" is specified

        # Jinja2 templating, handled using the python "jinja2" module
        file_loader = FileSystemLoader(script_dir + "/templates")
        env = Environment(loader=file_loader, trim_blocks=True, lstrip_blocks=True)

        if not os.path.isfile("./traefik/docker-compose.yml"):
            traefik_compose_template = env.get_template("traefik_compose.yml.j2")
            traefik_compose_output = traefik_compose_template.render(render_vars)
            with open(os.path.join(main_dir, "traefik/docker-compose.yml"), 'w') as traefik_compose_file:
                logging.info("Writing traefik docker-compose.yml")
                traefik_compose_file.truncate(0)
                traefik_compose_file.write(traefik_compose_output)

        if constants.CONF_ALL_HTTPS_PORT in config.keys():
            logging.info("HTTPS mode (with Authelia)")
            if constants.CONF_ALL_AUTHELIA_CONFIG_PATH in config.keys():
                if not config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH] == "" and os.path.isfile(os.path.abspath(config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH])):
                    try:
                        shutil.copy(os.path.abspath(config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH]), "./traefik/authelia/configuration.yml")
                    except Exception as exc:
                        logging.critical("Could not copy authelia configuration file")
                        sys.exit(exc)
                else:
                    logging.critical("Invalid authelia configuration path (%s)" % config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH])
                    sys.exit()

            # Path to the authelia users in the repo
            authelia_users_path = script_dir + "/files/authelia_users.yml"
            # Copy authelia "users" file
            if not os.path.isfile("./traefik/authelia/users.yml"):
                shutil.copy(authelia_users_path, "./traefik/authelia/users.yml")
        else:
            logging.info("HTTP mode (without Authelia)")

        # Create the mounts for the traefik and authelia services
        traefik_dir = os.path.abspath(os.path.join(main_dir, "traefik"))
        if not os.path.isdir(os.path.join(traefik_dir, "docker_data")):
            create_mounts(working_dir=traefik_dir, main_dir=main_dir)

        # Return to main directory
        os.chdir(main_dir)

def create_mounts(working_dir, main_dir):
        """
        Create the folders (volumes) required by a container (to see required volumes, check their compose file)

        :return:
        """

        # Change directory to create mount points for the container
        try:
            os.chdir(os.path.abspath(working_dir))
        except Exception as exc:
            logging.critical("Cannot access %s, exiting" % working_dir)
            sys.exit(exc)
        compose_yml = os.path.abspath("./docker-compose.yml")
        if not os.path.isfile(compose_yml):
            raise Exception("Could not find docker-compose.yml at %s" % compose_yml)

        with open(compose_yml) as f:
            compose = yaml.safe_load(f)

        if 'services' not in compose:
            raise Exception("Could not find services tag in docker-compose.yml")

        # Iterate over all services to find the "volumes" we need to create
        for service in compose['services']:
            if 'volumes' in compose['services'][service]:
                for volume in compose['services'][service]['volumes']:
                    # regex to match the volumes of the services
                    reg = re.match(r"^(\./[^:]+/):[^:]+(:\w+)?$", volume)
                    if reg:
                        vol_dir = os.path.abspath('./' + reg.group(1))
                        if not os.path.exists(vol_dir):
                            os.makedirs(vol_dir, exist_ok=True)
                    else:
                        reg = re.match(r"^(\./[^:]+):[^:]+(:\w+)?$", volume)
                        if reg:
                            vol_file = os.path.abspath('./' + reg.group(1))
                            vol_dir = os.path.dirname(vol_file)
                            if not os.path.exists(vol_dir):
                                os.makedirs(vol_dir, exist_ok=True)
                            if not os.path.exists(vol_file):
                                Path(vol_file).touch()

        # Go back to the "main" directory
        try:
            os.chdir(os.path.abspath(main_dir))
        except OSError as exc:
            logging.critical("Cannot access %s, exiting" % main_dir)
            sys.exit(exc)

def run_command(command, working_dir):
    subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=working_dir)

def run_docker_stack_deploy(service, working_dir):
    run_command(["docker", "stack", "deploy", "-c", "./docker-compose.yml", service], working_dir)

def deploy_stacks(input_list, main_dir, deploy_traefik):
    """
    This function first deploys/redeploys the traefik stack, then deploys/redeploys the organism stack, then redeploys the traefik stack
    This function is executed outside the "main" loop of input species

    :return:
    """

    main_dir = os.path.abspath(main_dir)
    traefik_dir = os.path.join(main_dir, "traefik")

    # Get species for which to deploy the stacks
    # Uses the get_unique_species_list method from utilities to deploy a stack only for the "species" level (i.e genus_species)
    to_deploy_species_li = utilities.get_unique_species_str_list(sp_dict_list=input_list)

    if deploy_traefik:
        # Create the swarm cluster if needed
        logging.info("Initializing docker swarm (adding node)")
        run_command(["docker", "swarm", "init"], main_dir)
        
        # Deploy traefik stack
        logging.info("Deploying traefik stack")
        run_docker_stack_deploy("traefik", traefik_dir)

    # Deploy individual species stacks
    for sp in to_deploy_species_li:
        sp_dir = os.path.join(main_dir, sp)
        logging.info("Deploying %s stack" % sp)
        run_docker_stack_deploy("{0}_{1}".format(sp.split("_")[0], sp.split("_")[1]), sp_dir)
        logging.info("Deployed %s stack" % sp)

    # Update traefik stack
    logging.info("Updating traefik stack")
    run_docker_stack_deploy("traefik", traefik_dir)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Deploy GGA containers")

    parser.add_argument("input",
                        type=str,
                        help="Input file (yml)")

    parser.add_argument("-v", "--verbose",
                        help="Increase output verbosity",
                        action="store_true")

    parser.add_argument("--config",
                        type=str,
                        help="Config path, default to 'examples/config.yml'")

    parser.add_argument("--main-directory",
                        type=str,
                        help="Where the stack containers will be located, defaults to current directory")

    parser.add_argument("--force-traefik",
                        help="Force overwrite traefik directory all docker-compose and conf files in the traefik and authelia directories (default=False)",
                        action="store_true")

    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    # Parsing the config file if provided, using the default config otherwise
    if args.config:
        config_file = os.path.abspath(args.config)
    else:
        config_file = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), constants.DEFAULT_CONFIG)
    config = utilities.parse_config(config_file)
    if (len(config.keys()) == 0):
        logging.error("Empty config dictionary")

    main_dir = None
    if not args.main_directory:
        main_dir = os.getcwd()
    else:
        main_dir = os.path.abspath(args.main_directory)

    sp_dict_list = utilities.parse_input(os.path.abspath(args.input))

    # Create traefik directory and compose files if needed or specified
    if args.force_traefik or not os.path.isdir(os.path.join(os.path.abspath(main_dir), "traefik")):
        make_traefik_compose_files(config=config, main_dir=main_dir)

    unique_sp_dict_list = utilities.get_unique_species_dict_list(sp_dict_list=sp_dict_list)
    sp_picture_dict = utilities.get_sp_picture(sp_dict_list=sp_dict_list)

    logging.info("Deploying stacks for organisms in input file %s" % args.input)
    for sp_dict in unique_sp_dict_list:

        # Init instance
        deploy_stack_for_current_organism = DeploySpeciesStack(parameters_dictionary=sp_dict)

        # Setting some of the instance attributes
        deploy_stack_for_current_organism.main_dir = main_dir
        deploy_stack_for_current_organism.species_dir = os.path.join(deploy_stack_for_current_organism.main_dir,
                                                                     deploy_stack_for_current_organism.genus_species +
                                                                     "/")

        # Parse the config yaml file
        deploy_stack_for_current_organism.config = config

        # Starting
        logging.info("gga_init.py called for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)

        # Make/update directory tree
        deploy_stack_for_current_organism.picture_path = sp_picture_dict[deploy_stack_for_current_organism.genus_species]
        deploy_stack_for_current_organism.make_directory_tree()
        logging.info("Successfully generated the directory tree for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)

        # Make compose files
        deploy_stack_for_current_organism.make_compose_files()
        logging.info("Successfully generated the docker-compose files for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)

    logging.info("Deploying stacks")
    if args.force_traefik:
        deploy_stacks(input_list=sp_dict_list, main_dir=main_dir, deploy_traefik=True)
    else:
        deploy_stacks(input_list=sp_dict_list, main_dir=main_dir, deploy_traefik=False)
    logging.info("All stacks deployed for organisms in input file %s" % args.input)