-
Arthur Le Bars authored8d36dd97
gga_init.py 19.53 KiB
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse
import os
import re
import subprocess
import logging
import sys
import yaml
import shutil
from pathlib import Path
from jinja2 import Template, Environment, FileSystemLoader
import utilities
import speciesData
"""
gga_init.py
Usage: $ python3 gga_init.py -i input_example.yml --config config.yml [OPTIONS]
TODO
- Exclude traefik dir tree creation and file writing from the loop (make make_dirs() an external func and write a func similar to make_compose_files()
for traefik and authelia as an external func)
"""
class DeploySpeciesStack(speciesData.SpeciesData):
"""
Child of SpeciesData
Contains methods and attributes to deploy a stack of services for a given organism, from creating/updating
the organism's directory tree to create the required docker-compose files and stack deployment
"""
def make_directory_tree(self):
"""
Generate the directory tree for an organism
:return:
"""
os.chdir(self.main_dir)
# Create the species main directory (name of the dir: genus_species)
try:
os.mkdir(self.species_dir)
except FileExistsError:
logging.info("Updating directory tree of %s" % self.genus_species)
try:
os.chdir(self.species_dir)
except OSError as exc:
logging.critical("Cannot access %s" % self.genus_species)
sys.exit(exc)
# Copy the custom banner to the species dir (banner used in tripal pages)
# If the path specified is invalid (because it's empty or is still the default demo one),
# use the default banner instead
if "banner_path" in self.config.keys():
if self.config["banner_path"] != "/path/to/banner" or self.config["banner_path"] != "":
try:
logging.debug("Custom banner path: %s" % self.config["banner_path"])
if os.path.isfile(os.path.abspath(self.config["banner_path"])):
shutil.copy(os.path.abspath(self.config["banner_path"]), "%s/banner.png" % self.species_dir)
except FileNotFoundError:
logging.warning("Specified banner not found (%s), using default banner instead" % self.config["banner_path"])
self.config.pop("banner_path", None)
else:
logging.debug("Using default banner for Tripal pages")
self.config.pop("banner_path", None)
else:
logging.debug("Using default banner for Tripal pages")
self.config.pop("banner_path", None)
# Create nginx dirs and write/re-write nginx conf
make_dirs(dir_paths_li=["./nginx", "./nginx/conf"])
try:
with open(os.path.abspath("./nginx/conf/default.conf"), 'w') as conf:
conf.write("server {\n\tlisten 80;\n\tserver_name ~.;\n\tlocation /download/ {\n\t\talias /project_data/; \n\t\tautoindex on;\n\t}\n}") # The species nginx conf
except OSError as exc:
logging.critical("Cannot edit NginX conf file")
sys.exit(exc)
# Return to main directory
os.chdir(self.main_dir)
logging.info("Directory tree generated for %s" % self.full_name)
def make_compose_files(self, entrypoint_mode):
"""
Create a formatted copy of the template compose file inside a species directory tree
:return:
"""
os.chdir(self.main_dir)
try:
os.chdir(self.species_dir)
except OSError as exc:
logging.critical("Cannot access %s" % self.species_dir)
sys.exit(exc)
# Jinja2 templating, handled using the python "jinja2" module
file_loader = FileSystemLoader(self.script_dir + "/templates")
env = Environment(loader=file_loader)
# We need a dict holding all key (variables) - values that needs to be replaced in the template as our rendering dict
# To do so we need both input file vars and config vars
# Create input file vars dict
input_vars = {"genus": self.genus_lowercase, "Genus": self.genus_uppercase, "species": self.species,
"genus_species": self.genus_species, "genus_species_strain_sex": self.full_name,
"strain": self.strain, "sex": self.sex, "Genus_species": self.genus_species[0].upper() + self.genus_species[1:]}
# Merge the two dicts
render_vars = {**self.config, **input_vars}
# Define entrypoint mode (http=web, https=webs)
if entrypoint_mode == "web" or entrypoint_mode == "webs":
entrypoint = {"entrypoint": entrypoint_mode}
else:
logging.critical("Entrypoint mode defined is invalid (%s defined, valid options : ['webs', 'web']")
sys.exit()
# Add entrypoint to render_vars dor templating
render_vars = {**render_vars, **entrypoint}
# Render the gspecies docker-compose file and write it
gspecies_compose_template = env.get_template("gspecies_compose_template.yml.j2")
gspecies_compose_output = gspecies_compose_template.render(render_vars)
with open(os.path.join(self.species_dir, "docker-compose.yml"), "w") as gspecies_compose_file:
logging.info("Writing %s docker-compose.yml" % self.genus_species)
gspecies_compose_file.truncate(0)
gspecies_compose_file.write(gspecies_compose_output)
# Create the volumes (directory) of the species docker-compose file
create_mounts(working_dir=".", main_dir=self.main_dir)
# Return to main directory
os.chdir(self.main_dir)
def make_orthology_compose_files(self):
"""
Create/update orthology compose files
:return:
"""
os.chdir(self.main_dir)
make_dirs["./orthology", "./orthology/src_data", "./orthology/src_data/genomes",
"./orthology/src_data/gff", "./orthology/src_data/newicks", "./orthology/src_data/proteomes"]
def make_dirs(dir_paths_li):
"""
Recursively create directories from a list of paths with a try-catch condition
:param dir_paths_li:
:return:
"""
created_dir_paths_li = []
for dir_path in dir_paths_li:
try:
os.mkdir(dir_path)
except FileExistsError:
logging.debug("%s directory already exists" % dir_path)
except PermissionError as exc:
logging.critical("Insufficient permission to create %s" % dir_path)
sys.exit(exc)
created_dir_paths_li.append(dir_path)
return created_dir_paths_li
def make_traefik_compose_files(config, main_dir, entrypoint_mode):
"""
Create or update the traefik directory, docker-compose file and authelia conf files
Only called when the argument "--traefik" is specified
:param config:
:param main_dir:
:return:
"""
script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
render_vars = config
# Define entrypoint mode (http=web, https=webs)
if entrypoint_mode == "web" or entrypoint_mode == "webs":
entrypoint = {"entrypoint": entrypoint_mode}
else:
logging.critical("Entrypoint mode defined is invalid (%s defined, valid options : ['webs', 'web']" % entrypoint_mode)
sys.exit()
# Add entrypoint to render_vars dor templating
render_vars = {**render_vars, **entrypoint}
os.chdir(main_dir)
# Create directory tree
make_dirs(["./traefik", "./traefik/authelia"])
# Render and try to write the traefik docker-compose file
# This new docker-compose file will not overwrite the one already present in the traefik dir
# unless the argument "--overwrite-all" is specified
# Jinja2 templating, handled using the python "jinja2" module
file_loader = FileSystemLoader(script_dir + "/templates")
env = Environment(loader=file_loader)
if not os.path.isfile("./traefik/docker-compose.yml"):
traefik_compose_template = env.get_template("traefik_compose_template.yml.j2")
traefik_compose_output = traefik_compose_template.render(render_vars)
with open(os.path.join(main_dir, "traefik/docker-compose.yml"), 'w') as traefik_compose_file:
logging.info("Writing traefik docker-compose.yml")
traefik_compose_file.truncate(0)
traefik_compose_file.write(traefik_compose_output)
if config["authelia_config_path"]:
if not config["authelia_config_path"] == "" or not config["authelia_config_path"] == "/path/to/authelia/config":
if os.path.isfile(os.path.abspath(config["authelia_config_path"])):
authelia_config_template = env.get_template(os.path.basename(config["authelia_config_path"]))
authelia_config_output = authelia_config_template.render(render_vars)
with open(os.path.join(main_dir, "traefik/authelia/configuration.yml"), 'w') as authelia_config_file:
logging.info("Writing authelia configuration.yml")
authelia_config_file.truncate(0)
authelia_config_file.write(authelia_config_output)
else:
logging.critical("Cannot find authelia configuration template path (%s)" % config["authelia_config_path"])
sys.exit()
# Path to the authelia users in the repo
authelia_users_path = script_dir + "/templates/authelia_users_template.yml"
# Copy authelia "users" file
if not os.path.isfile("./traefik/authelia/users.yml"):
shutil.copy(authelia_users_path, "./traefik/authelia/users.yml")
# Create the mounts for the traefik and authelia services
traefik_dir = os.path.abspath(os.path.join(main_dir, "traefik"))
if not os.path.isdir(os.path.join(traefik_dir, "docker_data")):
create_mounts(working_dir=traefik_dir, main_dir=main_dir)
# Return to main directory
os.chdir(main_dir)
def create_mounts(working_dir, main_dir):
"""
Create the folders (volumes) required by a container (to see required volumes, check their compose file)
:return:
"""
# Change directory to create mount points for the container
try:
os.chdir(os.path.abspath(working_dir))
except Exception as exc:
logging.critical("Cannot access %s, exiting" % working_dir)
sys.exit(exc)
compose_yml = os.path.abspath("./docker-compose.yml")
if not os.path.isfile(compose_yml):
raise Exception("Could not find docker-compose.yml at %s" % compose_yml)
with open(compose_yml) as f:
compose = yaml.safe_load(f)
if 'services' not in compose:
raise Exception("Could not find services tag in docker-compose.yml")
# Iterate over all services to find the "volumes" we need to create
for service in compose['services']:
if 'volumes' in compose['services'][service]:
for volume in compose['services'][service]['volumes']:
# regex to match the volumes of the services
reg = re.match(r"^(\./[^:]+/):[^:]+(:\w+)?$", volume)
if reg:
vol_dir = os.path.abspath('./' + reg.group(1))
if not os.path.exists(vol_dir):
os.makedirs(vol_dir, exist_ok=True)
else:
reg = re.match(r"^(\./[^:]+):[^:]+(:\w+)?$", volume)
if reg:
vol_file = os.path.abspath('./' + reg.group(1))
vol_dir = os.path.dirname(vol_file)
if not os.path.exists(vol_dir):
os.makedirs(vol_dir, exist_ok=True)
if not os.path.exists(vol_file):
Path(vol_file).touch()
# Go back to the "main" directory
try:
os.chdir(os.path.abspath(main_dir))
except OSError as exc:
logging.critical("Cannot access %s, exiting" % main_dir)
sys.exit(exc)
def deploy_stacks(input_list, main_dir, deploy_traefik):
"""
This function first deploys/redeploys the traefik stack, then deploys/redeploys the organism stack, then redeploys the traefik stack
This function is executed outside the "main" loop of input species
:return:
"""
main_dir = os.path.abspath(main_dir)
os.chdir(main_dir)
# Get species for which to deploy the stacks
# Uses the get_unique_species_list method from utilities to deploy a stack only for the "species" level (i.e genus_species)
to_deploy_species_li = utilities.get_unique_species_list(sp_dict_list=input_list)
if deploy_traefik:
# Create the swarm cluster if needed
logging.info("Initializing docker swarm (adding node)")
subprocess.call(["docker", "swarm", "init"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=main_dir)
# Deploy traefik stack
logging.info("Deploying traefik stack")
os.chdir("./traefik")
subprocess.call(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "traefik"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=".")
os.chdir(main_dir)
# Deploy individual species stacks
for sp in to_deploy_species_li:
os.chdir(sp)
subprocess.call(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "{0}_{1}".format(sp.split("_")[0], sp.split("_")[1])],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=".")
logging.info("Deployed %s stack" % sp)
os.chdir(main_dir)
# Update traefik stack
logging.info("Re-deploying traefik stack")
os.chdir("./traefik")
subprocess.call(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "traefik"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=".")
os.chdir(main_dir)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Automatic data loading in containers and interaction "
"with galaxy instances for GGA"
", following the protocol @ "
"http://gitlab.sb-roscoff.fr/abims/e-infra/gga")
parser.add_argument("input",
type=str,
help="Input file (yml)")
parser.add_argument("-v", "--verbose",
help="Increase output verbosity",
action="store_true")
parser.add_argument("--config",
type=str,
help="Config path, default to the 'config' file inside the script repository")
parser.add_argument("--main-directory",
type=str,
help="Where the stack containers will be located, defaults to current directory")
parser.add_argument("--traefik",
help="Initialize/wverwrite traefik directory all docker-compose and conf files in the traefik and authelia directories (default=False)",
action="store_true")
parser.add_argument("--http",
help="Use a HTTP authelia configuration",
action="store_true")
parser.add_argument("--https",
help="Use a HTTPS authelia configuration",
action="store_true")
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
# Parsing the config file if provided, using the default config otherwise
if not args.config:
args.config = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "config")
else:
args.config = os.path.abspath(args.config)
main_dir = None
if not args.main_directory:
main_dir = os.getcwd()
else:
main_dir = os.path.abspath(args.main_directory)
sp_dict_list = utilities.parse_input(os.path.abspath(args.input))
# Create traefik directory and compose files if specified
if args.traefik:
config = utilities.parse_config(args.config)
if args.https:
make_traefik_compose_files(config=config, main_dir=main_dir, entrypoint_mode="webs")
elif args.http:
make_traefik_compose_files(config=config, main_dir=main_dir, entrypoint_mode="web")
else:
logging.critical("Entrypoint mode is not defined (missing http or https argument")
sys.exit()
logging.info("Deploying stacks for organisms in input file %s" % args.input)
for sp_dict in sp_dict_list:
# Init instance
deploy_stack_for_current_organism = DeploySpeciesStack(parameters_dictionary=sp_dict)
# Setting some of the instance attributes
deploy_stack_for_current_organism.main_dir = main_dir
deploy_stack_for_current_organism.species_dir = os.path.join(deploy_stack_for_current_organism.main_dir,
deploy_stack_for_current_organism.genus_species +
"/")
# Parse the config yaml file
deploy_stack_for_current_organism.config = utilities.parse_config(args.config)
# Set the instance url attribute
for env_variable, value in deploy_stack_for_current_organism.config.items():
if env_variable == "hostname":
deploy_stack_for_current_organism.instance_url = value + \
deploy_stack_for_current_organism.genus_lowercase + \
"_" + deploy_stack_for_current_organism.species + \
"/galaxy/"
break
else:
deploy_stack_for_current_organism.instance_url = "http://localhost:8888/sp/{0}_{1}/galaxy/".format(
deploy_stack_for_current_organism.genus_lowercase,
deploy_stack_for_current_organism.species)
# Starting
logging.info("gga_init.py called for %s" % deploy_stack_for_current_organism.full_name)
# Make/update directory tree
deploy_stack_for_current_organism.make_directory_tree()
logging.info("Successfully generated the directory tree for %s" % deploy_stack_for_current_organism.full_name)
# Make compose files
if args.https:
deploy_stack_for_current_organism.make_compose_files(entrypoint_mode="webs")
elif args.http:
deploy_stack_for_current_organism.make_compose_files(entrypoint_mode="web")
else:
logging.critical("Entrypoint mode is not defined (missing http or https argument")
sys.exit()
logging.info("Successfully generated the docker-compose files for %s" % deploy_stack_for_current_organism.full_name)
logging.info("Deploying stacks")
if args.traefik:
deploy_stacks(input_list=sp_dict_list, main_dir=main_dir, deploy_traefik=True)
else:
deploy_stacks(input_list=sp_dict_list, main_dir=main_dir, deploy_traefik=False)
logging.info("All stacks deployed for organisms in input file %s" % args.input)