Release 2.0 (merge dev to master)

General

• Reorganize repository
• Fix verbosity mode
• Fix logs details
• Update README.md
• Update the "help" option in the scripts
• Update examples
• Factorize and optimize code
• Create file constants.py containing the constant variables

Input organisms.yml file

• Change root structure (a list instead of a dictionary, the name parameter is added to the species dictionary sp_dict)
• Add parameter main_species. Useful when there are multiple strains for the same species: set to yes to define the strain used as home for JBrowse
• Parameters species, genus, strain, sex, common will be sanitized (removing " ", "-", "(", ")", "'")
• Add jinja2 template

Input config file

• Change root structure (one-level dictionary)
• Add parameters authelia_secrets_env_path and authelia_db_postgres_password required for Authelia
• Change the name of some parameters (galaxy_persist_data, tripal_banner_path, jbrowse_menu_url)
• Remove unused apollo parameters (webapollo_user, webapollo_password)

Step "init"

• Update templates files
• Fix the copy of the specified banner
• Write galaxy_nginx.conf only if needed

Step "get data"

• Prefix data filenames with g_species_s
• Add the last modification date of the source file to the gff filename

Step "load data"

• Use Bioblend function upload_from_galaxy_fs() instead of upload_from_local() (to enable symlink, instead of copy)

Step "run workflow Phaeoexplorer"

• Development of a workflow loading fasta and gff files to Chado, Tripal and JBrowse, for one or two organisms of the same species

gga_init.py

@@ -9,12 +9,13 @@ import logging
 import sys
 import yaml
 import shutil
-
 from pathlib import Path
-from jinja2 import Template, Environment, FileSystemLoader
+from jinja2 import Environment, FileSystemLoader
 
 import utilities
 import speciesData
+import constants
+
 
 """ 
 gga_init.py
@@ -60,34 +61,31 @@ class DeploySpeciesStack(speciesData.SpeciesData):
         # Copy the custom banner to the species dir (banner used in tripal pages)
         # If the path specified is invalid (because it's empty or is still the default demo one),
         # use the default banner instead
-        if "banner_path" in self.config.keys():
-            if self.config["banner_path"] != "/path/to/banner" or self.config["banner_path"] != "":
-                try:
-                    logging.debug("Custom banner path: %s" % self.config["banner_path"])
-                    if os.path.isfile(os.path.abspath(self.config["banner_path"])):
-                        shutil.copy(os.path.abspath(self.config["banner_path"]), "%s/banner.png" % self.species_dir)
-                except FileNotFoundError:
-                    logging.warning("Specified banner not found (%s), using default banner instead" % self.config["banner_path"])
-                    self.config.pop("banner_path", None)
+        if constants.CONF_TRIPAL_BANNER_PATH in self.config.keys():
+            if not config[constants.CONF_TRIPAL_BANNER_PATH] == "" and os.path.isfile(os.path.abspath(config[constants.CONF_TRIPAL_BANNER_PATH])):
+                banner_dest_path = os.path.join(self.species_dir, os.path.abspath("banner.png"))
+                if not os.path.isfile(banner_dest_path) and not os.path.islink(banner_dest_path) and not os.path.samefile(os.path.abspath(config[constants.CONF_TRIPAL_BANNER_PATH]),banner_dest_path):
+                    os.symlink(os.path.abspath(self.config[constants.CONF_TRIPAL_BANNER_PATH]), banner_dest_path)
+                    logging.info("Custom banner added: symlink from %s" % self.config[constants.CONF_TRIPAL_BANNER_PATH])
             else:
-                logging.debug("Using default banner for Tripal pages")
-                self.config.pop("banner_path", None)
+                logging.debug("Using default banner for Tripal pages because %s is not valid in 'config' file" % constants.CONF_TRIPAL_BANNER_PATH)
+                self.config.pop(constants.CONF_TRIPAL_BANNER_PATH, None)
         else:
             logging.debug("Using default banner for Tripal pages")
-            self.config.pop("banner_path", None)
+            self.config.pop(constants.CONF_TRIPAL_BANNER_PATH, None)
 
         # Create nginx dirs and write/re-write nginx conf
         make_dirs(dir_paths_li=["./nginx", "./nginx/conf"])
         try:
             shutil.copy(os.path.join(self.script_dir, "files/nginx_download.conf"), os.path.abspath("./nginx/conf/default.conf"))
         except Exception as exc:
-            logging.critical("Could not copy nginx configuration file for %s" % self.full_name)
+            logging.critical("Could not copy nginx configuration file for %s %s", self.genus, self.species)
             logging.critical(exc)            
 
         # Return to main directory
         os.chdir(self.main_dir)
 
-        logging.info("Directory tree generated for %s" % self.full_name)
+        logging.info("Directory tree generated for %s %s", self.genus, self.species)
 
 
     def make_compose_files(self):
@@ -114,25 +112,30 @@ class DeploySpeciesStack(speciesData.SpeciesData):
         input_vars = {"genus": self.genus_lowercase, "Genus": self.genus_uppercase, "species": self.species,
                       "genus_species": self.genus_species, "genus_species_strain_sex": self.species_folder_name,
                       "genus_species_sex": "{0}_{1}_{2}".format(self.genus_lowercase, self.species.lower(), self.sex),
-                      "strain": self.strain, "sex": self.sex, "Genus_species": self.genus_species[0].upper() + self.genus_species[1:]}
+                      "strain": self.strain, "sex": self.sex, "Genus_species": self.genus_species[0].upper() + self.genus_species[1:],
+                      "blast": self.blast}
+        if (len(self.config.keys()) == 0):
+            logging.error("Empty config dictionary")
         # Merge the two dicts
         render_vars = {**self.config, **input_vars}
 
         # Render the gspecies docker-compose file and write it
-        gspecies_compose_template = env.get_template("gspecies_compose_template.yml.j2")
+        gspecies_compose_template = env.get_template("gspecies_compose.yml.j2")
         gspecies_compose_output = gspecies_compose_template.render(render_vars)
         with open(os.path.join(self.species_dir, "docker-compose.yml"), "w") as gspecies_compose_file:
             logging.info("Writing %s docker-compose.yml" % self.genus_species)
             gspecies_compose_file.truncate(0)
             gspecies_compose_file.write(gspecies_compose_output)
 
-
-        galaxy_nginx_conf_template = env.get_template("galaxy_nginx.conf.j2")
-        galaxy_nginx_conf_output = galaxy_nginx_conf_template.render(render_vars)
-        with open(os.path.join(self.main_dir, "galaxy_nginx.conf"), "w") as galaxy_nginx_conf_file:
-            logging.debug("Writing the galaxy_nginx.conf file for %s" % self.genus_species)
-            galaxy_nginx_conf_file.truncate(0)
-            galaxy_nginx_conf_file.write(galaxy_nginx_conf_output)
+        if not os.path.isfile(os.path.join(self.main_dir, "galaxy_nginx.conf")):
+            galaxy_nginx_conf_template = env.get_template("galaxy_nginx.conf.j2")
+            galaxy_nginx_conf_output = galaxy_nginx_conf_template.render(render_vars)
+            with open(os.path.join(self.main_dir, "galaxy_nginx.conf"), "w") as galaxy_nginx_conf_file:
+                logging.debug("Writing the galaxy_nginx.conf file for %s" % self.genus_species)
+                galaxy_nginx_conf_file.truncate(0)
+                galaxy_nginx_conf_file.write(galaxy_nginx_conf_output)
+        else:
+            logging.debug("galaxy_nginx.conf already exists")
 
         # Create the volumes (directory) of the species docker-compose file
         create_mounts(working_dir=".", main_dir=self.main_dir)
@@ -199,42 +202,36 @@ def make_traefik_compose_files(config, main_dir):
 
         # Jinja2 templating, handled using the python "jinja2" module
         file_loader = FileSystemLoader(script_dir + "/templates")
-        env = Environment(loader=file_loader)
+        env = Environment(loader=file_loader, trim_blocks=True, lstrip_blocks=True)
 
         if not os.path.isfile("./traefik/docker-compose.yml"):
-            traefik_compose_template = env.get_template("traefik_compose_template.yml.j2")
+            traefik_compose_template = env.get_template("traefik_compose.yml.j2")
             traefik_compose_output = traefik_compose_template.render(render_vars)
             with open(os.path.join(main_dir, "traefik/docker-compose.yml"), 'w') as traefik_compose_file:
                 logging.info("Writing traefik docker-compose.yml")
                 traefik_compose_file.truncate(0)
                 traefik_compose_file.write(traefik_compose_output)
 
-        if "authelia_config_path" in config.keys():
-            if not config["authelia_config_path"] == "" or not config["authelia_config_path"] == "/path/to/authelia/config":
-                if os.path.isfile(os.path.abspath(config["authelia_config_path"])):
+        if constants.CONF_ALL_HTTPS_PORT in config.keys():
+            logging.info("HTTPS mode (with Authelia)")
+            if constants.CONF_ALL_AUTHELIA_CONFIG_PATH in config.keys():
+                if not config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH] == "" and os.path.isfile(os.path.abspath(config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH])):
                     try:
-                        shutil.copy(os.path.abspath(config["authelia_config_path"]), "./traefik/authelia/configuration.yml")
+                        shutil.copy(os.path.abspath(config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH]), "./traefik/authelia/configuration.yml")
                     except Exception as exc:
                         logging.critical("Could not copy authelia configuration file")
                         sys.exit(exc)
-                    # authelia_config_template = env.get_template(os.path.basename(config["authelia_config_path"]))
-                    # authelia_config_output = authelia_config_template.render(render_vars)
-                    # with open(os.path.join(main_dir, "traefik/authelia/configuration.yml"), 'w') as authelia_config_file:
-                    #     logging.info("Writing authelia configuration.yml")
-                    #     authelia_config_file.truncate(0)
-                    #     authelia_config_file.write(authelia_config_output)
                 else:
-                    logging.critical("Cannot find authelia configuration path (%s)" % config["authelia_config_path"])
+                    logging.critical("Invalid authelia configuration path (%s)" % config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH])
                     sys.exit()
-            else:
-                logging.critical("Invalid authelia configuration path (%s)" % config["authelia_config_path"])
-                sys.exit()
 
-        # Path to the authelia users in the repo
-        authelia_users_path = script_dir + "/files/authelia_users.yml"
-        # Copy authelia "users" file
-        if not os.path.isfile("./traefik/authelia/users.yml"):
-            shutil.copy(authelia_users_path, "./traefik/authelia/users.yml")
+            # Path to the authelia users in the repo
+            authelia_users_path = script_dir + "/files/authelia_users.yml"
+            # Copy authelia "users" file
+            if not os.path.isfile("./traefik/authelia/users.yml"):
+                shutil.copy(authelia_users_path, "./traefik/authelia/users.yml")
+        else:
+            logging.info("HTTP mode (without Authelia)")
 
         # Create the mounts for the traefik and authelia services
         traefik_dir = os.path.abspath(os.path.join(main_dir, "traefik"))
@@ -244,7 +241,6 @@ def make_traefik_compose_files(config, main_dir):
         # Return to main directory
         os.chdir(main_dir)
 
-
 def create_mounts(working_dir, main_dir):
         """
         Create the folders (volumes) required by a container (to see required volumes, check their compose file)
@@ -295,6 +291,12 @@ def create_mounts(working_dir, main_dir):
             logging.critical("Cannot access %s, exiting" % main_dir)
             sys.exit(exc)
 
+def run_command(command, working_dir):
+    subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=working_dir)
+
+def run_docker_stack_deploy(service, working_dir):
+    run_command(["docker", "stack", "deploy", "-c", "./docker-compose.yml", service], working_dir)
+
 def deploy_stacks(input_list, main_dir, deploy_traefik):
     """
     This function first deploys/redeploys the traefik stack, then deploys/redeploys the organism stack, then redeploys the traefik stack
@@ -304,7 +306,7 @@ def deploy_stacks(input_list, main_dir, deploy_traefik):
     """
 
     main_dir = os.path.abspath(main_dir)
-    os.chdir(main_dir)
+    traefik_dir = os.path.join(main_dir, "traefik")
 
     # Get species for which to deploy the stacks
     # Uses the get_unique_species_list method from utilities to deploy a stack only for the "species" level (i.e genus_species)
@@ -313,38 +315,26 @@ def deploy_stacks(input_list, main_dir, deploy_traefik):
     if deploy_traefik:
         # Create the swarm cluster if needed
         logging.info("Initializing docker swarm (adding node)")
-        subprocess.call(["docker", "swarm", "init"],
-                        stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=main_dir)
+        run_command(["docker", "swarm", "init"], main_dir)
         
         # Deploy traefik stack
         logging.info("Deploying traefik stack")
-        os.chdir("./traefik")
-        subprocess.call(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "traefik"],
-                                                stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=".")
-        os.chdir(main_dir)
+        run_docker_stack_deploy("traefik", traefik_dir)
 
     # Deploy individual species stacks
     for sp in to_deploy_species_li:
-        os.chdir(sp)
+        sp_dir = os.path.join(main_dir, sp)
         logging.info("Deploying %s stack" % sp)
-        subprocess.call(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "{0}_{1}".format(sp.split("_")[0], sp.split("_")[1])],
-                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=".")
+        run_docker_stack_deploy("{0}_{1}".format(sp.split("_")[0], sp.split("_")[1]), sp_dir)
         logging.info("Deployed %s stack" % sp)
-        os.chdir(main_dir)
 
     # Update traefik stack
     logging.info("Updating traefik stack")
-    os.chdir("./traefik")
-    subprocess.call(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "traefik"],
-                                            stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=".")
-    os.chdir(main_dir)
+    run_docker_stack_deploy("traefik", traefik_dir)
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Automatic data loading in containers and interaction "
-                                                 "with galaxy instances for GGA"
-                                                 ", following the protocol @ "
-                                                 "http://gitlab.sb-roscoff.fr/abims/e-infra/gga")
+    parser = argparse.ArgumentParser(description="Deploy GGA containers")
 
     parser.add_argument("input",
                         type=str,
@@ -356,7 +346,7 @@ if __name__ == "__main__":
 
     parser.add_argument("--config",
                         type=str,
-                        help="Config path, default to the 'config' file inside the script repository")
+                        help="Config path, default to 'examples/config.yml'")
 
     parser.add_argument("--main-directory",
                         type=str,
@@ -374,10 +364,13 @@ if __name__ == "__main__":
         logging.basicConfig(level=logging.INFO)
 
     # Parsing the config file if provided, using the default config otherwise
-    if not args.config:
-        args.config = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "config")
+    if args.config:
+        config_file = os.path.abspath(args.config)
     else:
-        args.config = os.path.abspath(args.config)
+        config_file = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), constants.DEFAULT_CONFIG)
+    config = utilities.parse_config(config_file)
+    if (len(config.keys()) == 0):
+        logging.error("Empty config dictionary")
 
     main_dir = None
     if not args.main_directory:
@@ -389,7 +382,6 @@ if __name__ == "__main__":
 
     # Create traefik directory and compose files if needed or specified
     if args.force_traefik or not os.path.isdir(os.path.join(os.path.abspath(main_dir), "traefik")):
-        config = utilities.parse_config(args.config)
         make_traefik_compose_files(config=config, main_dir=main_dir)
 
     unique_sp_dict_list = utilities.get_unique_species_dict_list(sp_dict_list=sp_dict_list)
@@ -407,31 +399,18 @@ if __name__ == "__main__":
                                                                      "/")
 
         # Parse the config yaml file
-        deploy_stack_for_current_organism.config = utilities.parse_config(args.config)
-
-        # Set the instance url attribute
-        for env_variable, value in deploy_stack_for_current_organism.config.items():
-            if env_variable == "hostname":
-                deploy_stack_for_current_organism.instance_url = value + \
-                                                                 deploy_stack_for_current_organism.genus_lowercase + \
-                                                                 "_" + deploy_stack_for_current_organism.species + \
-                                                                 "/galaxy/"
-                break
-            else:
-                deploy_stack_for_current_organism.instance_url = "http://localhost:8888/sp/{0}_{1}/galaxy/".format(
-                    deploy_stack_for_current_organism.genus_lowercase,
-                    deploy_stack_for_current_organism.species)
-                                                                    
+        deploy_stack_for_current_organism.config = config
+
         # Starting
-        logging.info("gga_init.py called for %s" % deploy_stack_for_current_organism.full_name)
+        logging.info("gga_init.py called for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
 
         # Make/update directory tree
         deploy_stack_for_current_organism.make_directory_tree()
-        logging.info("Successfully generated the directory tree for %s" % deploy_stack_for_current_organism.full_name)
+        logging.info("Successfully generated the directory tree for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
 
         # Make compose files
         deploy_stack_for_current_organism.make_compose_files()
-        logging.info("Successfully generated the docker-compose files for %s" % deploy_stack_for_current_organism.full_name)
+        logging.info("Successfully generated the docker-compose files for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
 
     logging.info("Deploying stacks")
     if args.force_traefik: