#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse
import os
import re
import subprocess
import logging
import sys
import yaml
import shutil
from pathlib import Path
from jinja2 import Environment, FileSystemLoader
import utilities
import speciesData
import constants
"""
gga_init.py
Usage: $ python3 gga_init.py -i input_example.yml --config config.yml [OPTIONS]
TODO
- Exclude traefik dir tree creation and file writing from the loop (make make_dirs() an external func and write a func similar to make_compose_files()
for traefik and authelia as an external func)
"""
class DeploySpeciesStack(speciesData.SpeciesData):
"""
Child of SpeciesData
Contains methods and attributes to deploy a stack of services for a given organism, from creating/updating
the organism's directory tree to create the required docker-compose files and stack deployment
"""
def make_directory_tree(self):
"""
Generate the directory tree for an organism
:return:
"""
os.chdir(self.main_dir)
# Create the species main directory (name of the dir: genus_species)
try:
os.mkdir(self.species_dir)
except FileExistsError:
logging.info("Updating directory tree of %s" % self.genus_species)
try:
os.chdir(self.species_dir)
except OSError as exc:
logging.critical("Cannot access %s" % self.genus_species)
sys.exit(exc)
# Copy the custom banner to the species dir (banner used in tripal pages)
# If the path specified is invalid (because it's empty or is still the default demo one),
# use the default banner instead
if constants.CONF_TRIPAL_BANNER_PATH in self.config.keys():
if not config[constants.CONF_TRIPAL_BANNER_PATH] == "" and os.path.isfile(os.path.abspath(config[constants.CONF_TRIPAL_BANNER_PATH])):
banner_dest_path = os.path.join(self.species_dir, os.path.abspath("banner.png"))
if not os.path.isfile(banner_dest_path) and not os.path.islink(banner_dest_path):
os.symlink(os.path.abspath(self.config[constants.CONF_TRIPAL_BANNER_PATH]), banner_dest_path)
logging.info("Custom banner added: symlink from %s" % self.config[constants.CONF_TRIPAL_BANNER_PATH])
else:
logging.info("Banner already exists at %s. The banner defined in config is not added." % banner_dest_path)
else:
logging.debug("Using default banner for Tripal pages because %s is not valid in 'config' file" % constants.CONF_TRIPAL_BANNER_PATH)
self.config.pop(constants.CONF_TRIPAL_BANNER_PATH, None)
else:
logging.debug("Using default banner for Tripal pages")
self.config.pop(constants.CONF_TRIPAL_BANNER_PATH, None)
# Copy the organism picture for tripal
if self.picture_path is not None:
if os.path.isfile(self.picture_path):
picture_path_basename = os.path.basename(self.picture_path)
picture_path_filename, picture_path_extension = os.path.splitext(picture_path_basename)
if picture_path_extension == ".png" or picture_path_extension == ".jpg":
picture_dest_name = "species%s" % picture_path_extension
picture_dest_path = os.path.join(self.species_dir, picture_dest_name)
shutil.copy(self.picture_path, picture_dest_path)
else:
logging.error("Specified organism picture has wrong extension (must be '.png' or '.jpg'): {0}".format(self.picture_path))
else:
logging.error("Specified organism picture not found {0} for {1}".format(self.picture_path, self.genus_uppercase + " " + self.species))
# Create nginx dirs and write/re-write nginx conf
make_dirs(dir_paths_li=["./nginx", "./nginx/conf"])
try:
shutil.copy(os.path.join(self.script_dir, "files/nginx_download.conf"), os.path.abspath("./nginx/conf/default.conf"))
except Exception as exc:
logging.critical("Could not copy nginx configuration file for %s %s", self.genus, self.species)
logging.critical(exc)
# Return to main directory
os.chdir(self.main_dir)
logging.info("Directory tree generated for %s %s", self.genus, self.species)
def make_compose_files(self):
"""
Create a formatted copy of the template compose file inside a species directory tree
:return:
"""
os.chdir(self.main_dir)
try:
os.chdir(self.species_dir)
except OSError as exc:
logging.critical("Cannot access %s" % self.species_dir)
sys.exit(exc)
# Jinja2 templating, handled using the python "jinja2" module
file_loader = FileSystemLoader(self.script_dir + "/templates")
env = Environment(loader=file_loader, trim_blocks=True, lstrip_blocks=True)
# We need a dict holding all key (variables) - values that needs to be replaced in the template as our rendering dict
# To do so we need both input file vars and config vars
# Create input file vars dict
input_vars = {"genus": self.genus_lowercase, "Genus": self.genus_uppercase, "species": self.species,
"genus_species": self.genus_species, "genus_species_strain_sex": self.species_folder_name,
"genus_species_sex": "{0}_{1}_{2}".format(self.genus_lowercase, self.species_lowercase, self.sex),
"strain": self.strain, "sex": self.sex, "Genus_species": "{0} {1}".format(self.genus_uppercase, self.species_lowercase),
"blast": self.blast, "go": self.go, "picture_path": self.picture_path}
if (len(self.config.keys()) == 0):
logging.error("Empty config dictionary")
# Merge the two dicts
render_vars = {**self.config, **input_vars}
# Render the gspecies docker-compose file and write it
gspecies_compose_template = env.get_template("gspecies_compose.yml.j2")
gspecies_compose_output = gspecies_compose_template.render(render_vars)
with open(os.path.join(self.species_dir, "docker-compose.yml"), "w") as gspecies_compose_file:
logging.info("Writing %s docker-compose.yml" % self.genus_species)
gspecies_compose_file.truncate(0)
gspecies_compose_file.write(gspecies_compose_output)
if not os.path.isfile(os.path.join(self.main_dir, "galaxy_nginx.conf")):
galaxy_nginx_conf_template = env.get_template("galaxy_nginx.conf.j2")
galaxy_nginx_conf_output = galaxy_nginx_conf_template.render(render_vars)
with open(os.path.join(self.main_dir, "galaxy_nginx.conf"), "w") as galaxy_nginx_conf_file:
logging.debug("Writing the galaxy_nginx.conf file for %s" % self.genus_species)
galaxy_nginx_conf_file.truncate(0)
galaxy_nginx_conf_file.write(galaxy_nginx_conf_output)
else:
logging.debug("galaxy_nginx.conf already exists")
# Create the volumes (directory) of the species docker-compose file
create_mounts(working_dir=".", main_dir=self.main_dir)
# Return to main directory
os.chdir(self.main_dir)
def make_orthology_compose_files(self):
"""
Create/update orthology compose files
:return:
"""
os.chdir(self.main_dir)
make_dirs["./orthology", "./orthology/src_data", "./orthology/src_data/genomes",
"./orthology/src_data/gff", "./orthology/src_data/newicks", "./orthology/src_data/proteomes"]
def make_dirs(dir_paths_li):
"""
Recursively create directories from a list of paths with a try-catch condition
:param dir_paths_li:
:return:
"""
created_dir_paths_li = []
for dir_path in dir_paths_li:
try:
os.mkdir(dir_path)
except FileExistsError:
logging.debug("%s directory already exists" % dir_path)
except PermissionError as exc:
logging.critical("Insufficient permission to create %s" % dir_path)
sys.exit(exc)
created_dir_paths_li.append(dir_path)
return created_dir_paths_li
def make_traefik_compose_files(config, main_dir):
"""
Create or update the traefik directory, docker-compose file and authelia conf files
Only called when the argument "--traefik" is specified
:param config:
:param main_dir:
:return:
"""
script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
render_vars = config
os.chdir(main_dir)
# Create directory tree
make_dirs(["./traefik", "./traefik/authelia"])
# Render and try to write the traefik docker-compose file
# This new docker-compose file will not overwrite the one already present in the traefik dir
# unless the argument "--overwrite-all" is specified
# Jinja2 templating, handled using the python "jinja2" module
file_loader = FileSystemLoader(script_dir + "/templates")
env = Environment(loader=file_loader, trim_blocks=True, lstrip_blocks=True)
if not os.path.isfile("./traefik/docker-compose.yml"):
traefik_compose_template = env.get_template("traefik_compose.yml.j2")
traefik_compose_output = traefik_compose_template.render(render_vars)
with open(os.path.join(main_dir, "traefik/docker-compose.yml"), 'w') as traefik_compose_file:
logging.info("Writing traefik docker-compose.yml")
traefik_compose_file.truncate(0)
traefik_compose_file.write(traefik_compose_output)
if constants.CONF_ALL_HTTPS_PORT in config.keys():
logging.info("HTTPS mode (with Authelia)")
if constants.CONF_ALL_AUTHELIA_CONFIG_PATH in config.keys():
if not config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH] == "" and os.path.isfile(os.path.abspath(config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH])):
try:
shutil.copy(os.path.abspath(config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH]), "./traefik/authelia/configuration.yml")
except Exception as exc:
logging.critical("Could not copy authelia configuration file")
sys.exit(exc)
else:
logging.critical("Invalid authelia configuration path (%s)" % config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH])
sys.exit()
# Path to the authelia users in the repo
authelia_users_path = script_dir + "/files/authelia_users.yml"
# Copy authelia "users" file
if not os.path.isfile("./traefik/authelia/users.yml"):
shutil.copy(authelia_users_path, "./traefik/authelia/users.yml")
else:
logging.info("HTTP mode (without Authelia)")
# Create the mounts for the traefik and authelia services
traefik_dir = os.path.abspath(os.path.join(main_dir, "traefik"))
if not os.path.isdir(os.path.join(traefik_dir, "docker_data")):
create_mounts(working_dir=traefik_dir, main_dir=main_dir)
# Return to main directory
os.chdir(main_dir)
def create_mounts(working_dir, main_dir):
"""
Create the folders (volumes) required by a container (to see required volumes, check their compose file)
:return:
"""
# Change directory to create mount points for the container
try:
os.chdir(os.path.abspath(working_dir))
except Exception as exc:
logging.critical("Cannot access %s, exiting" % working_dir)
sys.exit(exc)
compose_yml = os.path.abspath("./docker-compose.yml")
if not os.path.isfile(compose_yml):
raise Exception("Could not find docker-compose.yml at %s" % compose_yml)
with open(compose_yml) as f:
compose = yaml.safe_load(f)
if 'services' not in compose:
raise Exception("Could not find services tag in docker-compose.yml")
# Iterate over all services to find the "volumes" we need to create
for service in compose['services']:
if 'volumes' in compose['services'][service]:
for volume in compose['services'][service]['volumes']:
# regex to match the volumes of the services
reg = re.match(r"^(\./[^:]+/):[^:]+(:\w+)?$", volume)
if reg:
vol_dir = os.path.abspath('./' + reg.group(1))
if not os.path.exists(vol_dir):
os.makedirs(vol_dir, exist_ok=True)
else:
reg = re.match(r"^(\./[^:]+):[^:]+(:\w+)?$", volume)
if reg:
vol_file = os.path.abspath('./' + reg.group(1))
vol_dir = os.path.dirname(vol_file)
if not os.path.exists(vol_dir):
os.makedirs(vol_dir, exist_ok=True)
if not os.path.exists(vol_file):
Path(vol_file).touch()
# Go back to the "main" directory
try:
os.chdir(os.path.abspath(main_dir))
except OSError as exc:
logging.critical("Cannot access %s, exiting" % main_dir)
sys.exit(exc)
def run_command(command, working_dir):
subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=working_dir)
def run_docker_stack_deploy(service, working_dir):
run_command(["docker", "stack", "deploy", "-c", "./docker-compose.yml", service], working_dir)
def deploy_stacks(input_list, main_dir, deploy_traefik):
"""
This function first deploys/redeploys the traefik stack, then deploys/redeploys the organism stack, then redeploys the traefik stack
This function is executed outside the "main" loop of input species
:return:
"""
main_dir = os.path.abspath(main_dir)
traefik_dir = os.path.join(main_dir, "traefik")
# Get species for which to deploy the stacks
# Uses the get_unique_species_list method from utilities to deploy a stack only for the "species" level (i.e genus_species)
to_deploy_species_li = utilities.get_unique_species_str_list(sp_dict_list=input_list)
if deploy_traefik:
# Create the swarm cluster if needed
logging.info("Initializing docker swarm (adding node)")
run_command(["docker", "swarm", "init"], main_dir)
# Deploy traefik stack
logging.info("Deploying traefik stack")
run_docker_stack_deploy("traefik", traefik_dir)
# Deploy individual species stacks
for sp in to_deploy_species_li:
sp_dir = os.path.join(main_dir, sp)
logging.info("Deploying %s stack" % sp)
run_docker_stack_deploy("{0}_{1}".format(sp.split("_")[0], sp.split("_")[1]), sp_dir)
logging.info("Deployed %s stack" % sp)
# Update traefik stack
logging.info("Updating traefik stack")
run_docker_stack_deploy("traefik", traefik_dir)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Deploy GGA containers")
parser.add_argument("input",
type=str,
help="Input file (yml)")
parser.add_argument("-v", "--verbose",
help="Increase output verbosity",
action="store_true")
parser.add_argument("--config",
type=str,
help="Config path, default to 'examples/config.yml'")
parser.add_argument("--main-directory",
type=str,
help="Where the stack containers will be located, defaults to current directory")
parser.add_argument("--force-traefik",
help="Force overwrite traefik directory all docker-compose and conf files in the traefik and authelia directories (default=False)",
action="store_true")
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
# Parsing the config file if provided, using the default config otherwise
if args.config:
config_file = os.path.abspath(args.config)
else:
config_file = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), constants.DEFAULT_CONFIG)
config = utilities.parse_config(config_file)
if (len(config.keys()) == 0):
logging.error("Empty config dictionary")
main_dir = None
if not args.main_directory:
main_dir = os.getcwd()
else:
main_dir = os.path.abspath(args.main_directory)
sp_dict_list = utilities.parse_input(os.path.abspath(args.input))
# Create traefik directory and compose files if needed or specified
if args.force_traefik or not os.path.isdir(os.path.join(os.path.abspath(main_dir), "traefik")):
make_traefik_compose_files(config=config, main_dir=main_dir)
unique_sp_dict_list = utilities.get_unique_species_dict_list(sp_dict_list=sp_dict_list)
sp_picture_dict = utilities.get_sp_picture(sp_dict_list=sp_dict_list)
logging.info("Deploying stacks for organisms in input file %s" % args.input)
for sp_dict in unique_sp_dict_list:
# Init instance
deploy_stack_for_current_organism = DeploySpeciesStack(parameters_dictionary=sp_dict)
# Setting some of the instance attributes
deploy_stack_for_current_organism.main_dir = main_dir
deploy_stack_for_current_organism.species_dir = os.path.join(deploy_stack_for_current_organism.main_dir,
deploy_stack_for_current_organism.genus_species +
"/")
# Parse the config yaml file
deploy_stack_for_current_organism.config = config
# Starting
logging.info("gga_init.py called for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
# Make/update directory tree
deploy_stack_for_current_organism.picture_path = sp_picture_dict[deploy_stack_for_current_organism.genus_species]
deploy_stack_for_current_organism.make_directory_tree()
logging.info("Successfully generated the directory tree for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
# Make compose files
deploy_stack_for_current_organism.make_compose_files()
logging.info("Successfully generated the docker-compose files for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
logging.info("Deploying stacks")
if args.force_traefik:
deploy_stacks(input_list=sp_dict_list, main_dir=main_dir, deploy_traefik=True)
else:
deploy_stacks(input_list=sp_dict_list, main_dir=main_dir, deploy_traefik=False)
logging.info("All stacks deployed for organisms in input file %s" % args.input)