Release_2.0 next

General

• Move constants to file constants.py
• Set default config file in scripts.
• Update description of scripts in main.
• Update README.md

Input config file

• Change config root to dict and update parse_config().
• Change some config parameters name (galaxy_persist_data, tripal_banner_path, jbrowse_menu_url), update example config file and compose templates.
• Remove apollo config variables (webapollo_user, webapollo_password) in config file and in gspecies compose template, because apollo service not deployed here.

Utilities.py

• Factorize parse_input() et parse_config().
• Get input yml keys from constants (update organisms.yml.j2 and add create_org_param_dict_from_constants())

gspecies_compose.yml

• set ENABLE_WIKI and ENABLE_APOLLO to 0 in gspecies compose template
• fix ENABLE_BLAST value in gspecies compose template

gga_get_data.py

• Remove useless variable datasets_to_get

gga_init.py

• replace subprocess.call() with subprocess.run(). Factorize run_command() and run_docker_stack_deploy()
• fix banner copy

gga_init.py

@@ -9,12 +9,13 @@ import logging
 import sys
 import yaml
 import shutil
-
 from pathlib import Path
 from jinja2 import Environment, FileSystemLoader
 
 import utilities
 import speciesData
+import constants
+
 
 """ 
 gga_init.py
@@ -60,18 +61,18 @@ class DeploySpeciesStack(speciesData.SpeciesData):
         # Copy the custom banner to the species dir (banner used in tripal pages)
         # If the path specified is invalid (because it's empty or is still the default demo one),
         # use the default banner instead
-        if "banner_path" in self.config.keys():
-            if not config["banner_path"] == "" and os.path.isfile(os.path.abspath(config["banner_path"])):
+        if constants.CONF_TRIPAL_BANNER_PATH in self.config.keys():
+            if not config[constants.CONF_TRIPAL_BANNER_PATH] == "" and os.path.isfile(os.path.abspath(config[constants.CONF_TRIPAL_BANNER_PATH])):
                 banner_dest_path = os.path.join(self.species_dir, os.path.abspath("banner.png"))
-                if not os.path.isfile(banner_dest_path) or not os.path.samefile(os.path.abspath(config["banner_path"]),banner_dest_path):
-                    os.symlink(os.path.abspath(self.config["banner_path"]), banner_dest_path)
-                    logging.info("Custom banner added: symlink from %s" % self.config["banner_path"])
+                if not os.path.isfile(banner_dest_path) and not os.path.islink(banner_dest_path) and not os.path.samefile(os.path.abspath(config[constants.CONF_TRIPAL_BANNER_PATH]),banner_dest_path):
+                    os.symlink(os.path.abspath(self.config[constants.CONF_TRIPAL_BANNER_PATH]), banner_dest_path)
+                    logging.info("Custom banner added: symlink from %s" % self.config[constants.CONF_TRIPAL_BANNER_PATH])
             else:
-                logging.debug("Using default banner for Tripal pages")
-                self.config.pop("banner_path", None)
+                logging.debug("Using default banner for Tripal pages because %s is not valid in 'config' file" % constants.CONF_TRIPAL_BANNER_PATH)
+                self.config.pop(constants.CONF_TRIPAL_BANNER_PATH, None)
         else:
             logging.debug("Using default banner for Tripal pages")
-            self.config.pop("banner_path", None)
+            self.config.pop(constants.CONF_TRIPAL_BANNER_PATH, None)
 
         # Create nginx dirs and write/re-write nginx conf
         make_dirs(dir_paths_li=["./nginx", "./nginx/conf"])
@@ -111,7 +112,10 @@ class DeploySpeciesStack(speciesData.SpeciesData):
         input_vars = {"genus": self.genus_lowercase, "Genus": self.genus_uppercase, "species": self.species,
                       "genus_species": self.genus_species, "genus_species_strain_sex": self.species_folder_name,
                       "genus_species_sex": "{0}_{1}_{2}".format(self.genus_lowercase, self.species.lower(), self.sex),
-                      "strain": self.strain, "sex": self.sex, "Genus_species": self.genus_species[0].upper() + self.genus_species[1:]}
+                      "strain": self.strain, "sex": self.sex, "Genus_species": self.genus_species[0].upper() + self.genus_species[1:],
+                      "blast": self.blast}
+        if (len(self.config.keys()) == 0):
+            logging.error("Empty config dictionary")
         # Merge the two dicts
         render_vars = {**self.config, **input_vars}
 
@@ -130,6 +134,8 @@ class DeploySpeciesStack(speciesData.SpeciesData):
                 logging.debug("Writing the galaxy_nginx.conf file for %s" % self.genus_species)
                 galaxy_nginx_conf_file.truncate(0)
                 galaxy_nginx_conf_file.write(galaxy_nginx_conf_output)
+        else:
+            logging.debug("galaxy_nginx.conf already exists")
 
         # Create the volumes (directory) of the species docker-compose file
         create_mounts(working_dir=".", main_dir=self.main_dir)
@@ -206,22 +212,26 @@ def make_traefik_compose_files(config, main_dir):
                 traefik_compose_file.truncate(0)
                 traefik_compose_file.write(traefik_compose_output)
 
-        if "authelia_config_path" in config.keys():
-            if not config["authelia_config_path"] == "" and os.path.isfile(os.path.abspath(config["authelia_config_path"])):
-                try:
-                    shutil.copy(os.path.abspath(config["authelia_config_path"]), "./traefik/authelia/configuration.yml")
-                except Exception as exc:
-                    logging.critical("Could not copy authelia configuration file")
-                    sys.exit(exc)
-            else:
-                logging.critical("Invalid authelia configuration path (%s)" % config["authelia_config_path"])
-                sys.exit()
-
-        # Path to the authelia users in the repo
-        authelia_users_path = script_dir + "/files/authelia_users.yml"
-        # Copy authelia "users" file
-        if not os.path.isfile("./traefik/authelia/users.yml"):
-            shutil.copy(authelia_users_path, "./traefik/authelia/users.yml")
+        if constants.CONF_ALL_HTTPS_PORT in config.keys():
+            logging.info("HTTPS mode (with Authelia)")
+            if constants.CONF_ALL_AUTHELIA_CONFIG_PATH in config.keys():
+                if not config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH] == "" and os.path.isfile(os.path.abspath(config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH])):
+                    try:
+                        shutil.copy(os.path.abspath(config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH]), "./traefik/authelia/configuration.yml")
+                    except Exception as exc:
+                        logging.critical("Could not copy authelia configuration file")
+                        sys.exit(exc)
+                else:
+                    logging.critical("Invalid authelia configuration path (%s)" % config[constants.CONF_ALL_AUTHELIA_CONFIG_PATH])
+                    sys.exit()
+
+            # Path to the authelia users in the repo
+            authelia_users_path = script_dir + "/files/authelia_users.yml"
+            # Copy authelia "users" file
+            if not os.path.isfile("./traefik/authelia/users.yml"):
+                shutil.copy(authelia_users_path, "./traefik/authelia/users.yml")
+        else:
+            logging.info("HTTP mode (without Authelia)")
 
         # Create the mounts for the traefik and authelia services
         traefik_dir = os.path.abspath(os.path.join(main_dir, "traefik"))
@@ -231,7 +241,6 @@ def make_traefik_compose_files(config, main_dir):
         # Return to main directory
         os.chdir(main_dir)
 
-
 def create_mounts(working_dir, main_dir):
         """
         Create the folders (volumes) required by a container (to see required volumes, check their compose file)
@@ -282,6 +291,12 @@ def create_mounts(working_dir, main_dir):
             logging.critical("Cannot access %s, exiting" % main_dir)
             sys.exit(exc)
 
+def run_command(command, working_dir):
+    subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=working_dir)
+
+def run_docker_stack_deploy(service, working_dir):
+    run_command(["docker", "stack", "deploy", "-c", "./docker-compose.yml", service], working_dir)
+
 def deploy_stacks(input_list, main_dir, deploy_traefik):
     """
     This function first deploys/redeploys the traefik stack, then deploys/redeploys the organism stack, then redeploys the traefik stack
@@ -291,7 +306,7 @@ def deploy_stacks(input_list, main_dir, deploy_traefik):
     """
 
     main_dir = os.path.abspath(main_dir)
-    os.chdir(main_dir)
+    traefik_dir = os.path.join(main_dir, "traefik")
 
     # Get species for which to deploy the stacks
     # Uses the get_unique_species_list method from utilities to deploy a stack only for the "species" level (i.e genus_species)
@@ -300,31 +315,22 @@ def deploy_stacks(input_list, main_dir, deploy_traefik):
     if deploy_traefik:
         # Create the swarm cluster if needed
         logging.info("Initializing docker swarm (adding node)")
-        subprocess.call(["docker", "swarm", "init"],
-                        stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=main_dir)
+        run_command(["docker", "swarm", "init"], main_dir)
         
         # Deploy traefik stack
         logging.info("Deploying traefik stack")
-        os.chdir("./traefik")
-        subprocess.call(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "traefik"],
-                                                stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=".")
-        os.chdir(main_dir)
+        run_docker_stack_deploy("traefik", traefik_dir)
 
     # Deploy individual species stacks
     for sp in to_deploy_species_li:
-        os.chdir(sp)
+        sp_dir = os.path.join(main_dir, sp)
         logging.info("Deploying %s stack" % sp)
-        subprocess.call(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "{0}_{1}".format(sp.split("_")[0], sp.split("_")[1])],
-                                           stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=".")
+        run_docker_stack_deploy("{0}_{1}".format(sp.split("_")[0], sp.split("_")[1]), sp_dir)
         logging.info("Deployed %s stack" % sp)
-        os.chdir(main_dir)
 
     # Update traefik stack
     logging.info("Updating traefik stack")
-    os.chdir("./traefik")
-    subprocess.call(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "traefik"],
-                                            stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=".")
-    os.chdir(main_dir)
+    run_docker_stack_deploy("traefik", traefik_dir)
 
 
 if __name__ == "__main__":
@@ -340,7 +346,7 @@ if __name__ == "__main__":
 
     parser.add_argument("--config",
                         type=str,
-                        help="Config path, default to the 'config' file inside the script repository")
+                        help="Config path, default to 'examples/config.yml'")
 
     parser.add_argument("--main-directory",
                         type=str,
@@ -357,8 +363,14 @@ if __name__ == "__main__":
     else:
         logging.basicConfig(level=logging.INFO)
 
-    config_file = os.path.abspath(args.config)
+    # Parsing the config file if provided, using the default config otherwise
+    if args.config:
+        config_file = os.path.abspath(args.config)
+    else:
+        config_file = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), constants.DEFAULT_CONFIG)
     config = utilities.parse_config(config_file)
+    if (len(config.keys()) == 0):
+        logging.error("Empty config dictionary")
 
     main_dir = None
     if not args.main_directory:
@@ -391,7 +403,6 @@ if __name__ == "__main__":
 
         # Starting
         logging.info("gga_init.py called for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
-        logging.debug("Jbrowse url to %s %s %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species, deploy_stack_for_current_organism.sex, deploy_stack_for_current_organism.strain)
 
         # Make/update directory tree
         deploy_stack_for_current_organism.make_directory_tree()