#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse
import os
import re
import subprocess
import logging
import sys
import yaml
import shutil
from pathlib import Path
from jinja2 import Template, Environment, FileSystemLoader
import utilities
import speciesData
"""
gga_init.py
Usage: $ python3 gga_init.py -i input_example.yml --config config.yml [OPTIONS]
"""
class DeploySpeciesStack(speciesData.SpeciesData):
"""
Child of SpeciesData
Contains methods and attributes to deploy a stack of services for a given organism, from creating/updating
the organism's directory tree to create the required docker-compose files and stack deployment
"""
def make_directory_tree(self):
"""
Generate the directory tree for an organism and move datasets into src_data
:return:
"""
os.chdir(self.main_dir)
# Create the species main directory (name of the dir: genus_species)
try:
os.mkdir(self.species_dir)
except FileExistsError:
logging.info("Updating directory tree of %s" % self.genus_species)
try:
os.chdir(self.species_dir)
except OSError as exc:
logging.critical("Cannot access %s" % self.genus_species)
sys.exit(exc)
# Copy the custom banner to the species dir (banner used in tripal pages)
# If the path specified is invalid (because it's empty or is still the default demo one),
# use the default banner instead
if "banner_path" in self.config.keys():
if self.config["banner_path"] != "/path/to/banner" or self.config["banner_path"] != "":
try:
logging.debug("Custom banner path: %s" % self.config["banner_path"])
if os.path.isfile(os.path.abspath(self.config["banner_path"])):
shutil.copy(os.path.abspath(self.config["banner_path"]), "%s/banner.png" % self.species_dir)
except FileNotFoundError:
logging.warning("Specified banner not found (%s), using default banner instead" % self.config["banner_path"])
self.config.pop("banner_path", None)
else:
logging.debug("Using default banner for Tripal pages")
self.config.pop("banner_path", None)
# Create nginx dirs and write/re-write nginx conf
self.make_dirs(dir_paths_li=["./nginx", "./nginx/conf"])
try:
with open(os.path.abspath("./nginx/conf/default.conf"), 'w') as conf:
conf.write("server {\n\tlisten 80;\n\tserver_name ~.;\n\tlocation /download/ {\n\t\talias /project_data/; \n\t\tautoindex on;\n\t}\n}") # The species nginx conf
except OSError as exc:
logging.critical("Cannot edit NginX conf file")
sys.exit(exc)
# Creation (or updating) of the src_data directory tree
try:
os.mkdir("./src_data")
except FileExistsError:
logging.debug("src_data folder already exist for %s" % self.full_name)
except PermissionError as exc:
logging.critical("Insufficient permission to create src_data directory tree")
sys.exit(exc)
# List of all the directories to create in src_data
src_data_dirs_li = ["./src_data", "./src_data/annotation", "./src_data/genome", "./src_data/tracks",
"./src_data/annotation/%s" % self.species_folder_name,
"./src_data/genome/%s" % self.species_folder_name,
"./src_data/annotation/{0}/OGS{1}/".format(self.species_folder_name, self.ogs_version),
"./src_data/genome/{0}/v{1}".format(self.species_folder_name, self.genome_version)]
self.make_dirs(dir_paths_li=src_data_dirs_li)
# Return to main directory
os.chdir(self.main_dir)
logging.info("Directory tree generated for %s" % self.full_name)
@staticmethod
def make_dirs(dir_paths_li):
"""
Recursively create directories from a list of paths with a try-catch condition
:param dir_paths_li:
:return:
"""
created_dir_paths_li = []
for dir_path in dir_paths_li:
try:
os.mkdir(dir_path)
except FileExistsError:
logging.debug("%s directory already exists" % dir_path)
except PermissionError as exc:
logging.critical("Insufficient permission to create %s" % dir_path)
sys.exit(exc)
created_dir_paths_li.append(dir_path)
return created_dir_paths_li
def make_compose_files(self, force=False):
"""
Create a formatted copy of the template compose file inside a species directory tree
:return:
"""
os.chdir(self.main_dir)
try:
os.chdir(self.species_dir)
except OSError as exc:
logging.critical("Cannot access %s" % self.species_dir)
sys.exit(exc)
# Jinja2 templating, handled using the python "jinja2" module
file_loader = FileSystemLoader(self.script_dir + "/templates")
env = Environment(loader=file_loader)
# We need a dict holding all key (variables) - values that needs to be replaced in the template as our rendering dict
# To do so we need both input file vars and config vars
# Create input file vars dict
input_vars = {"genus": self.genus_lowercase, "Genus": self.genus_uppercase, "species": self.species,
"genus_species": self.genus_species, "genus_species_strain_sex": self.full_name,
"strain": self.strain, "sex": self.sex, "Genus_species": self.genus_species[0].upper() + self.genus_species[1:]}
# Merge the two dicts
render_vars = {**self.config, **input_vars}
# Render the gspecies docker-compose file and write it
gspecies_compose_template = env.get_template("gspecies_compose_template.yml.j2")
gspecies_compose_output = gspecies_compose_template.render(render_vars)
with open(os.path.join(self.species_dir, "docker-compose.yml"), "w") as gspecies_compose_file:
logging.info("Writing %s docker-compose.yml" % self.genus_species)
gspecies_compose_file.truncate(0)
gspecies_compose_file.write(gspecies_compose_output)
# Create the volumes (directory) of the species docker-compose file
self.create_mounts(working_dir=".")
# Proceed to the traefik and authelia directories
os.chdir(self.main_dir)
self.make_dirs(["./traefik", "./traefik/authelia"])
# Render and try to write the traefik docker-compose file
# This new docker-compose file will not overwrite the one already present in the traefik dir
# unless the argument "--overwrite-all" is specified
if not os.path.isfile("./traefik/docker-compose.yml") or force:
traefik_compose_template = env.get_template("traefik_compose_template.yml.j2")
traefik_compose_output = traefik_compose_template.render(render_vars)
with open(os.path.join(self.main_dir, "docker-compose.yml"), 'w') as traefik_compose_file:
logging.info("Writing traefik docker-compose.yml")
traefik_compose_file.truncate(0)
traefik_compose_file.write(traefik_compose_output)
if self.config["authelia_config_path"]:
if not self.config["authelia_config_path"] == "" or not self.config["authelia_config_path"] == "/path/to/authelia/config":
if os.path.isfile(os.path.abspath(self.config["authelia_config_path"])):
try:
shutil.copy(os.path.abspath(self.config["authelia_config_path"]), "./traefik/authelia")
except Exception as exc:
logging.critical("Cannot copy custom Authelia config file (%s)" % self.config["authelia_config_path"])
sys.exit(exc)
else:
logging.critical("Custom Authelia config file not found (%s)" % self.config["authelia_config_path"])
# Path to the authelia users in the repo
authelia_users_path = self.script_dir + "/templates/authelia_users_template.yml"
# Copy authelia "users" file
if not os.path.isfile("./traefik/authelia/users.yml") or force:
shutil.copy(authelia_users_path, "./traefik/authelia/users.yml")
# Create the mounts for the traefik and authelia services
traefik_dir = os.path.abspath(os.path.join(self.main_dir, "traefik"))
if not os.path.isdir(os.path.join(traefik_dir, "docker_data")) or force:
self.create_mounts(working_dir=traefik_dir)
# Return to main directory
os.chdir(self.main_dir)
def create_mounts(self, working_dir):
"""
Create the folders (volumes) required by a container (to see required volumes, check their compose file)
:return:
"""
# Change directory to create mount points for the container
try:
os.chdir(os.path.abspath(working_dir))
except Exception as exc:
logging.critical("Cannot access %s, exiting" % working_dir)
sys.exit(exc)
compose_yml = os.path.abspath("./docker-compose.yml")
if not os.path.isfile(compose_yml):
raise Exception("Could not find docker-compose.yml at %s" % compose_yml)
with open(compose_yml) as f:
compose = yaml.safe_load(f)
if 'services' not in compose:
raise Exception("Could not find services tag in docker-compose.yml")
# Iterate over all services to find the "volumes" we need to create
for service in compose['services']:
if 'volumes' in compose['services'][service]:
for volume in compose['services'][service]['volumes']:
# regex to match the volumes of the services
reg = re.match(r"^(\./[^:]+/):[^:]+(:\w+)?$", volume)
if reg:
vol_dir = os.path.abspath('./' + reg.group(1))
if not os.path.exists(vol_dir):
os.makedirs(vol_dir, exist_ok=True)
else:
reg = re.match(r"^(\./[^:]+):[^:]+(:\w+)?$", volume)
if reg:
vol_file = os.path.abspath('./' + reg.group(1))
vol_dir = os.path.dirname(vol_file)
if not os.path.exists(vol_dir):
os.makedirs(vol_dir, exist_ok=True)
if not os.path.exists(vol_file):
Path(vol_file).touch()
# Go back to the "main" directory
try:
os.chdir(os.path.abspath(self.main_dir))
except OSError as exc:
logging.critical("Cannot access %s, exiting" % self.main_dir)
sys.exit(exc)
def deploy_stack(self, input_list):
"""
Call the script "deploy.sh" used to initiliaze the swarm cluster if needed and
launch/update the current organism's stack
This script first try to deploy the traefik stack, then deploy the organism stack, then update the traefik stack
The stacks are updated if already deployed
:return:
"""
to_deploy_species_li = []
# # Create our swarm cluster if it doesn't exist
# subprocess.Popen(["docker", "swarm", "init"],
# stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.main_dir)
#
# # Deploy/update the stack for the current species
# subprocess.Popen(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "{0}_{1}".format(self.genus_lowercase, self.species)],
# stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.main_dir)
# Launch and update docker stacks
# noinspection PyArgumentList
# deploy_stacks_popen = subprocess.Popen(["sh", self.script_dir + "/deploy.sh", self.genus_species,
# self.main_dir + "/traefik"],
# stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
# universal_newlines=True)
#
# for stdout_line in iter(deploy_stacks_popen.stdout.readline, ""):
# if "daemon" in stdout_line: # Ignore swarm init error output
# pass
# else:
# logging.info("\t%s" % stdout_line.strip())
# deploy_stacks_popen.stdout.close()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Automatic data loading in containers and interaction "
"with galaxy instances for GGA"
", following the protocol @ "
"http://gitlab.sb-roscoff.fr/abims/e-infra/gga")
parser.add_argument("input",
type=str,
help="Input file (yml)")
parser.add_argument("-v", "--verbose",
help="Increase output verbosity",
action="store_true")
parser.add_argument("--config",
type=str,
help="Config path, default to the 'config' file inside the script repository")
parser.add_argument("--main-directory",
type=str,
help="Where the stack containers will be located, defaults to current directory")
parser.add_argument("--overwrite-all",
help="Overwrite all docker-compose and conf files in the traefik and authelia directories (default=False)",
action="store_true")
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
# Parsing the config file if provided, using the default config otherwise
if not args.config:
args.config = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "config")
else:
args.config = os.path.abspath(args.config)
main_dir = None
if not args.main_directory:
main_dir = os.getcwd()
else:
main_dir = os.path.abspath(args.main_directory)
sp_dict_list = utilities.parse_input(os.path.abspath(args.input))
print(sp_dict_list)
utilities.get_species_to_deploy(sp_dict_list=sp_dict_list)
logging.info("Deploying stacks for organisms in input file %s" % args.input)
for sp_dict in sp_dict_list:
# Init instance
deploy_stack_for_current_organism = DeploySpeciesStack(parameters_dictionary=sp_dict)
# Setting some of the instance attributes
deploy_stack_for_current_organism.main_dir = main_dir
deploy_stack_for_current_organism.species_dir = os.path.join(deploy_stack_for_current_organism.main_dir,
deploy_stack_for_current_organism.genus_species +
"/")
# Parse the config yaml file
deploy_stack_for_current_organism.config = utilities.parse_config(args.config)
# Set the instance url attribute
for env_variable, value in deploy_stack_for_current_organism.config.items():
if env_variable == "hostname":
deploy_stack_for_current_organism.instance_url = value + \
deploy_stack_for_current_organism.genus_lowercase + \
"_" + deploy_stack_for_current_organism.species + \
"/galaxy/"
break
else:
deploy_stack_for_current_organism.instance_url = "http://localhost:8888/sp/{0}_{1}/galaxy/".format(
deploy_stack_for_current_organism.genus_lowercase,
deploy_stack_for_current_organism.species)
# Starting
logging.info("gga_init.py called for %s" % deploy_stack_for_current_organism.full_name)
# Make/update directory tree
deploy_stack_for_current_organism.make_directory_tree()
logging.info("Successfully generated the directory tree for %s" % deploy_stack_for_current_organism.full_name)
# Make compose files
deploy_stack_for_current_organism.make_compose_files(force=args.overwrite_all)
logging.info("Successfully generated the docker-compose files for %s" % deploy_stack_for_current_organism.full_name)
# Deploy the stack
logging.info("Deploying stack for %s..." % deploy_stack_for_current_organism.full_name)
# deploy_stack_for_current_organism.deploy_stack()
logging.info("Successfully deployed stack for %s" % deploy_stack_for_current_organism.full_name)
logging.info("Stack deployed for %s" % deploy_stack_for_current_organism.full_name)
# TODO: if GENUS°1 == GENUS°2 AND SP°1 == SP°2 --> skip init, continue to next item and only deploy once the loop is done
# TODO: reload traefik outside loop
logging.info("All stacks deployed for organisms in input file %s" % args.input)