#!/usr/bin/python # -*- coding: utf-8 -*- import argparse import os import subprocess import logging import sys import utilities import shutil import speciesData """ gga_init.py Usage: $ python3 gga_init.py -i example.yml [OPTIONS] """ class DeploySpeciesStack(speciesData.SpeciesData): """ Child of SpeciesData Contains methods and attributes to deploy a stack of services for a given organism, from creating/updating the organism's directory tree to create the required docker-compose files """ def make_directory_tree(self): """ Generate the directory tree for an organism and move datasets into src_data :return: """ os.chdir(self.main_dir) try: os.mkdir(self.species_dir) logging.info("Making directory tree for %s" % self.full_name) except FileExistsError: logging.info("Updating directory tree for %s" % self.full_name) try: os.chdir(self.species_dir) working_dir = os.getcwd() except OSError: logging.critical("Cannot access " + self.species_dir + ", run with higher privileges") sys.exit() try: os.mkdir("./nginx/") os.mkdir("./nginx/conf") with open(os.path.abspath("./nginx/conf/default.conf"), 'w') as conf: conf.write("server {\n\tlisten 80;\n\tserver_name ~.;\n\tlocation /download/ {\n\t\talias /project_data/; \n\t\tautoindex on;\n\t}\n}") # The species nginx conf except FileExistsError: logging.debug("NginX conf exists") organism_annotation_dir, organism_genome_dir = None, None # Creation (or updating) of the src_data directory tree # Depth 0 try: os.mkdir("./src_data") except FileExistsError: logging.debug("src_data folder already exist for %s" % self.full_name) except PermissionError: logging.critical("Insufficient permission to create src_data directory tree") sys.exit() # Depth 1 try: os.mkdir("./src_data/annotation") os.mkdir("./src_data/genome") os.mkdir("./src_data/tracks") except FileExistsError: logging.debug("Depth 1 src_data folder(s) already exist for %s" % self.full_name) except PermissionError: logging.critical("Insufficient permission to create src_data directory tree") sys.exit() # Depth 2 try: os.mkdir("./src_data/annotation/" + self.species_folder_name) os.mkdir("./src_data/genome/" + self.species_folder_name) except FileExistsError: logging.debug("Depth 2 src_data folder(s) already exist for %s" % self.full_name) except PermissionError: logging.critical("Insufficient permission to create src_data directory tree") sys.exit() # Depth 3 try: os.mkdir("./src_data/annotation/" + self.species_folder_name + "/OGS" + self.ogs_version) os.mkdir("./src_data/genome/" + self.species_folder_name + "/v" + self.genome_version) organism_annotation_dir = os.path.abspath("./src_data/annotation/" + self.species_folder_name + "/OGS" + self.genome_version) organism_genome_dir = os.path.abspath("./src_data/genome/" + self.species_folder_name + "/v" + self.genome_version) except FileExistsError: logging.debug("Depth 3 src_data folder(s) already exist for %s" % self.full_name) except PermissionError: logging.critical("Insufficient permission to create src_data directory tree") sys.exit() # Return to main_dir os.chdir(self.main_dir) def make_compose_files(self): """ Create a formatted copy of the template compose file inside a species directory tree :return: """ os.chdir(self.main_dir) try: os.chdir(self.species_dir) except OSError: logging.critical("Cannot access " + self.species_dir) sys.exit(0) # Path to the templates used to generate the custom docker-compose files for an input species stack_template_path = self.script_dir + "/templates/compose_template.yml" traefik_template_path = self.script_dir + "/templates/traefik.yml" authelia_config_path = self.script_dir + "/templates/authelia_config.yml" authelia_users_path = self.script_dir + "/templates/authelia_users.yml" if self.sex and self.strain: genus_species_strain_sex = "_".join([self.genus.lower(), self.species, self.strain, self.sex]) else: genus_species_strain_sex = "{0}_{1}".format(self.genus.lower(), self.species) with open(stack_template_path, 'r') as infile: organism_content = list() for line in infile: # Replace placeholders in the compose file and append line to output organism_content.append( line.replace("genus_species", str(self.genus.lower() + "_" + self.species)).replace("Genus species", str(self.genus_uppercase + " " + self.species)).replace("Genus/species", str(self.genus_uppercase + "/" + self.species)).replace("gspecies", str(self.genus.lower()[0] + self.species)).replace("genus_species_strain_sex", genus_species_strain_sex)) # Write output compose file with open("./docker-compose.yml", 'w') as outfile: outfile.truncate(0) # Delete file content for line in organism_content: # Replace env variables by those in the config file for env_variable, value in self.config.items(): if env_variable in line: line = line.replace(env_variable, value) break outfile.write(line) # Call create_mounts.py (replace subprocess.DEVNULL by subprocess.PIPE to get script stdout and stderr back) subprocess.call(["python3", self.script_dir + "/create_mounts.py"], cwd=self.species_dir, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) # Create mounts for the containers try: shutil.copy(os.path.join(self.script_dir, "galaxy_data_libs_SI.py"), "../galaxy_data_libs_SI.py") except FileExistsError: logging.debug("galxy_data_libs_SI.py already exists") try: os.mkdir("../traefik") os.mkdir("../traefik/authelia") shutil.copy(authelia_config_path, "../traefik/authelia/configuration.yml") shutil.copy(authelia_users_path, "../traefik/authelia/users.yml") subprocess.call(["python3", self.script_dir + "/create_mounts.py"], cwd=self.species_dir, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) # Create mounts for the containers except FileExistsError: logging.debug("Traefik directory already exists: %s" % os.path.abspath("../traefik")) try: shutil.copy(traefik_template_path, "../traefik/docker-compose.yml") except FileExistsError: logging.debug("Traefik compose file already exists: %s" % os.path.abspath("../traefik/docker-compose.yml")) subprocess.call(["python3", self.script_dir + "/create_mounts.py"], cwd=self.species_dir) os.chdir(self.main_dir) def deploy_stack(self): """ Call the script "deploy.sh" used to initiliaze the swarm cluster if needed and launch/update the current organism's stack This script first try to deploy the traefik stack, then deploy the organism stack, then update the traefik stack The stacks are updated if already deployed :return: """ # Launch and update docker stacks # noinspection PyArgumentList deploy_stacks_popen = subprocess.Popen(["sh", self.script_dir + "/deploy.sh", self.genus_species, self.main_dir + "/traefik"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) for stdout_line in iter(deploy_stacks_popen.stdout.readline, ""): if "daemon" in stdout_line: # Ignore swarm init error output pass else: logging.info("\t%s" % stdout_line.strip()) deploy_stacks_popen.stdout.close() if __name__ == "__main__": parser = argparse.ArgumentParser(description="Automatic data loading in containers and interaction " "with galaxy instances for GGA" ", following the protocol @ " "http://gitlab.sb-roscoff.fr/abims/e-infra/gga") parser.add_argument("input", type=str, help="Input file (yml)") parser.add_argument("-v", "--verbose", help="Increase output verbosity", action="store_true") parser.add_argument("--config", type=str, help="Config path, default to the 'config' file inside the script repository") parser.add_argument("--main-directory", type=str, help="Where the stack containers will be located, defaults to working directory") args = parser.parse_args() if args.verbose: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) # Parsing the config file if provided, using the default config otherwise if not args.config: args.config = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "config") else: args.config = os.path.abspath(args.config) if not args.main_directory: args.main_directory = os.getcwd() else: args.main_directory = os.path.abspath(args.main_directory) sp_dict_list = utilities.parse_input(os.path.abspath(args.input)) logging.info("Deploying stacks for organisms in input file %s" % args.input) for sp_dict in sp_dict_list: # Init instance deploy_stack_for_current_organism = DeploySpeciesStack(parameters_dictionary=sp_dict) # Setting some of the instance attributes deploy_stack_for_current_organism.main_dir = os.getcwd() deploy_stack_for_current_organism.species_dir = os.path.join(deploy_stack_for_current_organism.main_dir, deploy_stack_for_current_organism.genus_species + "/") deploy_stack_for_current_organism.config = utilities.parse_config(args.config) # Starting logging.info("gga_init.py called for %s" % deploy_stack_for_current_organism.full_name) # Make/update directory tree deploy_stack_for_current_organism.make_directory_tree() logging.info("Successfully generated the directory tree for %s" % deploy_stack_for_current_organism.full_name) # Make compose files deploy_stack_for_current_organism.make_compose_files() logging.info("Successfully generated the docker-compose files for %s" % deploy_stack_for_current_organism.full_name) # Deploy the stack logging.info("Deploying stack for %s..." % deploy_stack_for_current_organism.full_name) deploy_stack_for_current_organism.deploy_stack() logging.info("Successfully deployed stack for %s" % deploy_stack_for_current_organism.full_name) logging.info("Stack deployed for %s" % deploy_stack_for_current_organism.full_name) logging.info("All stacks deployed for organisms in input file %s" % args.input)