#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse
import os
import re
import subprocess
import logging
import sys
import yaml
import shutil
from pathlib import Path
from jinja2 import Environment, FileSystemLoader
import utilities
import species_data
import constants
"""
gga_init.py
Usage: $ python3 gga_init.py -i input_example.yml --config config.yml [OPTIONS]
TODO
- Exclude traefik dir tree creation and file writing from the loop (make make_dirs() an external func and write a func similar to make_compose_files()
for traefik and authelia as an external func)
"""
class DeploySpeciesStack(species_data.SpeciesData):
"""
Child of SpeciesData
Contains methods and attributes to deploy a stack of services for a given organism, from creating/updating
the organism's directory tree to create the required docker-compose files and stack deployment
"""
def __init__(self, parameters_dictionary):
self.picture_path = None
self.jbrowse_links = None
super().__init__(parameters_dictionary)
def make_directory_tree(self):
"""
Generate the directory tree for an organism
:return:
"""
os.chdir(self.main_dir)
# Create the species main directory (name of the dir: genus_species)
try:
os.mkdir(self.species_dir)
except FileExistsError:
logging.info("Updating directory tree of %s" % self.genus_species)
try:
os.chdir(self.species_dir)
except OSError as exc:
logging.critical("Cannot access %s" % self.genus_species)
sys.exit(exc)
# Copy the custom banner to the species dir (banner used in tripal pages)
# If the path specified is invalid (because it's empty or is still the default demo one),
# use the default banner instead
if constants.CONF_TRIPAL_BANNER_PATH in self.config.keys():
if not config[constants.CONF_TRIPAL_BANNER_PATH] == "" and os.path.isfile(os.path.abspath(config[constants.CONF_TRIPAL_BANNER_PATH])):
banner_dest_path = os.path.join(self.species_dir, os.path.abspath("banner.png"))
if not os.path.isfile(banner_dest_path) and not os.path.islink(banner_dest_path):
os.symlink(os.path.abspath(self.config[constants.CONF_TRIPAL_BANNER_PATH]), banner_dest_path)
logging.info("Custom banner added: symlink from %s" % self.config[constants.CONF_TRIPAL_BANNER_PATH])
else:
logging.info("Banner already exists at %s. The banner defined in config is not added." % banner_dest_path)
else:
logging.debug("Using default banner for Tripal pages because %s is not valid in 'config' file" % constants.CONF_TRIPAL_BANNER_PATH)
self.config.pop(constants.CONF_TRIPAL_BANNER_PATH, None)
else:
logging.debug("Using default banner for Tripal pages")
self.config.pop(constants.CONF_TRIPAL_BANNER_PATH, None)
# Copy the organism picture for tripal
if self.picture_path is not None:
if os.path.isfile(self.picture_path):
picture_path_basename = os.path.basename(self.picture_path)
picture_path_filename, picture_path_extension = os.path.splitext(picture_path_basename)
if picture_path_extension == ".png" or picture_path_extension == ".jpg":
picture_dest_name = "species%s" % picture_path_extension
picture_dest_path = os.path.join(self.species_dir, picture_dest_name)
shutil.copy(self.picture_path, picture_dest_path)
logging.info("Add picture %s" % self.picture_path)
else:
logging.error("Specified organism picture has wrong extension (must be '.png' or '.jpg'): {0}".format(self.picture_path))
else:
logging.error("Specified organism picture not found {0} for {1}".format(self.picture_path, self.genus_uppercase + " " + self.species))
# Create nginx dirs and write/re-write nginx conf
make_dirs(dir_paths_li=["./nginx", "./nginx/conf"])
try:
shutil.copy(os.path.join(self.script_dir, "files/nginx_download.conf"), os.path.abspath("./nginx/conf/default.conf"))
except Exception as exc:
logging.critical("Could not copy nginx configuration file for %s %s", self.genus, self.species)
logging.critical(exc)
# Return to main directory
os.chdir(self.main_dir)
logging.info("Directory tree generated for %s %s", self.genus, self.species)
def make_compose_files(self):
"""
Create a formatted copy of the template compose file inside a species directory tree
:return:
"""
os.chdir(self.main_dir)
try:
os.chdir(self.species_dir)
except OSError as exc:
logging.critical("Cannot access %s" % self.species_dir)
sys.exit(exc)
# Jinja2 templating, handled using the python "jinja2" module
file_loader = FileSystemLoader(self.script_dir + "/templates")
env = Environment(loader=file_loader, trim_blocks=True, lstrip_blocks=True)
# We need a dict holding all key (variables) - values that needs to be replaced in the template as our rendering dict
# To do so we need both input file vars and config vars
# Create input file vars dict
input_vars = {"genus": self.genus_lowercase, "Genus": self.genus_uppercase, "species": self.species,
"genus_species": self.genus_species, "jbrowse_dataset_id": self.species_folder_name, "jbrowse_links": self.jbrowse_links,
"genus_species_sex": "{0}_{1}_{2}".format(self.genus_lowercase, self.species_lowercase, self.sex),
"strain": self.strain, "sex": self.sex, "Genus_species": "{0} {1}".format(self.genus_uppercase, self.species_lowercase),
"blast": self.blast, "go": self.go, "picture_path": self.picture_path}
if len(self.config.keys()) == 0:
logging.error("Empty config dictionary")
# Merge the two dicts
render_vars = {**self.config, **input_vars}
# Render the gspecies docker-compose file and write it
gspecies_compose_template = env.get_template("gspecies_compose.yml.j2")
gspecies_compose_output = gspecies_compose_template.render(render_vars)
with open(os.path.join(self.species_dir, "docker-compose.yml"), "w") as gspecies_compose_file:
logging.info("Writing %s docker-compose.yml" % self.genus_species)
gspecies_compose_file.truncate(0)
gspecies_compose_file.write(gspecies_compose_output)
if not os.path.isfile(os.path.join(self.main_dir, "galaxy_nginx.conf")):
galaxy_nginx_conf_template = env.get_template("galaxy_nginx.conf.j2")
galaxy_nginx_conf_output = galaxy_nginx_conf_template.render(render_vars)
with open(os.path.join(self.main_dir, "galaxy_nginx.conf"), "w") as galaxy_nginx_conf_file:
logging.debug("Writing the galaxy_nginx.conf file for %s" % self.genus_species)
galaxy_nginx_conf_file.truncate(0)
galaxy_nginx_conf_file.write(galaxy_nginx_conf_output)
else:
logging.debug("galaxy_nginx.conf already exists")
# Create the volumes (directory) of the species docker-compose file
create_mounts(working_dir=".", main_dir=self.main_dir)
# Return to main directory
os.chdir(self.main_dir)
def make_dirs(dir_paths_li):
"""
Recursively create directories from a list of paths with a try-catch condition
:param dir_paths_li:
:return:
"""
created_dir_paths_li = []
for dir_path in dir_paths_li:
try:
os.mkdir(dir_path)
except FileExistsError:
logging.debug("%s directory already exists" % dir_path)
except PermissionError as exc:
logging.critical("Insufficient permission to create %s" % dir_path)
sys.exit(exc)
created_dir_paths_li.append(dir_path)
return created_dir_paths_li
def make_traefik_compose_files(config, main_dir):
"""
Create or update the traefik directory, docker-compose file and authelia conf files
Only called when the argument "--traefik" is specified
:param config:
:param main_dir:
:return:
"""
script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
render_vars = config
os.chdir(main_dir)
# Create directory tree
make_dirs(["./traefik", "./traefik/authelia"])
# Render and try to write the traefik docker-compose file
# This new docker-compose file will not overwrite the one already present in the traefik dir
# unless the argument "--overwrite-all" is specified
# Jinja2 templating, handled using the python "jinja2" module
file_loader = FileSystemLoader(script_dir + "/templates")
env = Environment(loader=file_loader, trim_blocks=True, lstrip_blocks=True)
if not os.path.isfile("./traefik/docker-compose.yml"):
traefik_compose_template = env.get_template("traefik_compose.yml.j2")
traefik_compose_output = traefik_compose_template.render(render_vars)
with open(os.path.join(main_dir, "traefik/docker-compose.yml"), 'w') as traefik_compose_file:
logging.info("Writing traefik docker-compose.yml")
traefik_compose_file.truncate(0)
traefik_compose_file.write(traefik_compose_output)
if constants.CONF_ALL_HTTPS_PORT in config.keys():
logging.info("HTTPS mode (with Authelia)")
if constants.CONF_ALL_AUTHELIA_CONFIG_PATH in config.keys():
if not config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH] == "" and os.path.isfile(os.path.abspath(config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH])):
try:
shutil.copy(os.path.abspath(config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH]), "./traefik/authelia/configuration.yml")
except Exception as exc:
logging.critical("Could not copy authelia configuration file")
sys.exit(exc)
else:
logging.critical("Invalid authelia configuration path (%s)" % config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH])
sys.exit()
# Path to the authelia users in the repo
authelia_users_path = script_dir + "/files/authelia_users.yml"
# Copy authelia "users" file
if not os.path.isfile("./traefik/authelia/users.yml"):
shutil.copy(authelia_users_path, "./traefik/authelia/users.yml")
else:
logging.info("HTTP mode (without Authelia)")
# Create the mounts for the traefik and authelia services
traefik_dir = os.path.abspath(os.path.join(main_dir, "traefik"))
if not os.path.isdir(os.path.join(traefik_dir, "docker_data")):
create_mounts(working_dir=traefik_dir, main_dir=main_dir)
# Return to main directory
os.chdir(main_dir)
def create_mounts(working_dir, main_dir):
"""
Create the folders (volumes) required by a container (to see required volumes, check their compose file)
:return:
"""
# Change directory to create mount points for the container
try:
os.chdir(os.path.abspath(working_dir))
except Exception as exc:
logging.critical("Cannot access %s, exiting" % working_dir)
sys.exit(exc)
compose_yml = os.path.abspath("./docker-compose.yml")
if not os.path.isfile(compose_yml):
raise Exception("Could not find docker-compose.yml at %s" % compose_yml)
with open(compose_yml) as f:
compose = yaml.safe_load(f)
if 'services' not in compose:
raise Exception("Could not find services tag in docker-compose.yml")
# Iterate over all services to find the "volumes" we need to create
for service in compose['services']:
if 'volumes' in compose['services'][service]:
for volume in compose['services'][service]['volumes']:
# regex to match the volumes of the services
reg = re.match(r"^(\./[^:]+/):[^:]+(:\w+)?$", volume)
if reg:
vol_dir = os.path.abspath('./' + reg.group(1))
if not os.path.exists(vol_dir):
os.makedirs(vol_dir, exist_ok=True)
else:
reg = re.match(r"^(\./[^:]+):[^:]+(:\w+)?$", volume)
if reg:
vol_file = os.path.abspath('./' + reg.group(1))
vol_dir = os.path.dirname(vol_file)
if not os.path.exists(vol_dir):
os.makedirs(vol_dir, exist_ok=True)
if not os.path.exists(vol_file):
Path(vol_file).touch()
# Go back to the "main" directory
try:
os.chdir(os.path.abspath(main_dir))
except OSError as exc:
logging.critical("Cannot access %s, exiting" % main_dir)
sys.exit(exc)
def run_command(command, working_dir):
subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=working_dir)
def run_docker_stack_deploy(service, working_dir):
run_command(["docker", "stack", "deploy", "-c", "./docker-compose.yml", service], working_dir)
def deploy_stacks(input_list, main_dir, deploy_traefik):
"""
This function first deploys/redeploys the traefik stack, then deploys/redeploys the organism stack, then redeploys the traefik stack
This function is executed outside the "main" loop of input species
:return:
"""
main_dir = os.path.abspath(main_dir)
traefik_dir = os.path.join(main_dir, "traefik")
# Get species for which to deploy the stacks
# Uses the get_unique_species_list method from utilities to deploy a stack only for the "species" level (i.e genus_species)
to_deploy_species_li = utilities.get_unique_species_str_list(sp_dict_list=input_list)
if deploy_traefik:
# Create the swarm cluster if needed
logging.info("Initializing docker swarm (adding node)")
run_command(["docker", "swarm", "init"], main_dir)
# Deploy traefik stack
logging.info("Deploying traefik stack")
run_docker_stack_deploy("traefik", traefik_dir)
# Deploy individual species stacks
for sp in to_deploy_species_li:
sp_dir = os.path.join(main_dir, sp)
logging.info("Deploying %s stack" % sp)
run_docker_stack_deploy("{0}_{1}".format(sp.split("_")[0], sp.split("_")[1]), sp_dir)
logging.info("Deployed %s stack" % sp)
# Update traefik stack
logging.info("Updating traefik stack")
run_docker_stack_deploy("traefik", traefik_dir)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Deploy GGA containers")
parser.add_argument("input",
type=str,
help="Input file (yml)")
parser.add_argument("-v", "--verbose",
help="Increase output verbosity",
action="store_true")
parser.add_argument("--config",
type=str,
help="Config path, default to 'examples/config.yml'")
parser.add_argument("--main-directory",
type=str,
help="Where the stack containers will be located, defaults to current directory")
parser.add_argument("--force-traefik",
help="Force overwrite traefik directory all docker-compose and conf files in the traefik and authelia directories (default=False)",
action="store_true")
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
# Parsing the config file if provided, using the default config otherwise
if args.config:
config_file = os.path.abspath(args.config)
else:
config_file = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), constants.DEFAULT_CONFIG)
config = utilities.parse_config(config_file)
if len(config.keys()) == 0:
logging.error("Empty config dictionary")
main_dir = None
if not args.main_directory:
main_dir = os.getcwd()
else:
main_dir = os.path.abspath(args.main_directory)
sp_dict_list = utilities.parse_input(os.path.abspath(args.input))
#TODO: create SpeciesData objects in utilities.parse_input()
org_list = []
for sp_dict in sp_dict_list:
org = DeploySpeciesStack(parameters_dictionary=sp_dict)
org_list.append(org)
# Create traefik directory and compose files if needed or specified
if args.force_traefik or not os.path.isdir(os.path.join(os.path.abspath(main_dir), "traefik")):
make_traefik_compose_files(config=config, main_dir=main_dir)
unique_sp_dict_list = utilities.get_unique_species_dict_list(sp_dict_list=sp_dict_list)
sp_picture_dict = utilities.get_sp_picture(sp_dict_list=sp_dict_list)
sp_jbrowse_links_dict = utilities.get_sp_jbrowse_links(org_list=org_list)
logging.info("Deploying stacks for organisms in input file %s" % args.input)
for sp_dict in unique_sp_dict_list:
# Init instance
deploy_stack_for_current_organism = DeploySpeciesStack(parameters_dictionary=sp_dict)
# Setting some of the instance attributes
deploy_stack_for_current_organism.main_dir = main_dir
deploy_stack_for_current_organism.species_dir = os.path.join(deploy_stack_for_current_organism.main_dir,
deploy_stack_for_current_organism.genus_species +
"/")
# Parse the config yaml file
deploy_stack_for_current_organism.config = config
# Starting
logging.info("gga_init.py called for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
# Make/update directory tree
deploy_stack_for_current_organism.picture_path = sp_picture_dict[deploy_stack_for_current_organism.genus_species]
deploy_stack_for_current_organism.make_directory_tree()
logging.info("Successfully generated the directory tree for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
# Make compose files
deploy_stack_for_current_organism.jbrowse_links = sp_jbrowse_links_dict[deploy_stack_for_current_organism.genus_species]
deploy_stack_for_current_organism.make_compose_files()
logging.info("Successfully generated the docker-compose files for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
# logging.info("Deploying stacks")
# if args.force_traefik:
# deploy_stacks(input_list=sp_dict_list, main_dir=main_dir, deploy_traefik=True)
# else:
# deploy_stacks(input_list=sp_dict_list, main_dir=main_dir, deploy_traefik=False)
# logging.info("All stacks deployed for organisms in input file %s" % args.input)