README.md

@@ -22,7 +22,7 @@ Traefik is a reverse proxy which allows to direct HTTP traffic to various Docker
 The Traefik dashboard is deployed at [https://hostname/traefik/](https://hostname/traefik/)
 
 Authelia is an authentication agent, which can be plugged to an LDAP server, and that Traefik can you to check permissions to access services.
-The authentication layer is optional. If used, the config file needs the variables `https_port`, `auth_hostname`, `authelia_config_path`.
+The authentication layer is optional. If used, the config file needs the variables `https_port`, `authentication_domain_name`, `authelia_config_path`.
 
 Authelia is accessed automatically by Traefik to check permissions everytime someones wants to access a page. 
 If the user is not logged in, he is redirected to the authelia portal. 

examples/authelia_config_example.yml → examples/authelia_config.yml

@@ -16,7 +16,7 @@ log_level: info
 # The secret used to generate JWT tokens when validating user identity by
 # email confirmation.
 # This secret can also be set using the env variables AUTHELIA_JWT_SECRET
-jwt_secret: XXXXXXXXXXXXXXXXX
+#jwt_secret: XXXXXXXXXXXXXXXXX
 
 # Default redirection URL
 #
@@ -82,7 +82,7 @@ authentication_backend:
 #    skip_verify: false
 
     # The base dn for every entries
-#    base_dn: dc=genouest,dc=org
+#    base_dn: dc=domain,dc=org
 
     # The attribute holding the username of the user. This attribute is used to populate
     # the username in the session information. It was introduced due to #561 to handle case
@@ -196,7 +196,7 @@ access_control:
   # Default policy can either be 'bypass', 'one_factor', 'two_factor' or 'deny'.
   # It is the policy applied to any resource if there is no policy to be applied
   # to the user.
-  default_policy: bypass
+  default_policy: deny
 
   rules:
     # The login portal is freely accessible (redirectino loop otherwise)
@@ -213,12 +213,9 @@ access_control:
     - domain: localhost
       resources:
         - "^/traefik/.*$"
+        - "^/api/.*$"
       policy: one_factor
       subject: "group:ldap_admin"
-    - domain: localhost
-      resources:
-        - "^/traefik/.*$"
-      policy: deny
 
     # All galaxies are restricted to a group from ldap
     - domain: localhost
@@ -237,22 +234,17 @@ access_control:
         - "^/sp/genus_species/.*$"
       policy: one_factor
       subject: "group:gspecies"
-    - domain: localhost
-      resources:
-        - "^/sp/genus_species/.*$"
-      policy: deny
-
 
 # Configuration of session cookies
 #
 # The session cookies identify the user once logged in.
 session:
   # The name of the session cookie. (default: authelia_session).
-  name: authelia_replaceme_session
+  name: authelia_session
 
   # The secret to encrypt the session data. This is only used with Redis.
   # This secret can also be set using the env variables AUTHELIA_SESSION_SECRET
-  secret: WXXXXXXXXXXXXXXXXXXXcXXXXXXXXXXXXXX
+#  secret: WXXXXXXXXXXXXXXXXXXXcXXXXXXXXXXXXXX
 
   # The time in seconds before the cookie expires and session is reset.
   expiration: 3600000 # 1000 hour
@@ -271,7 +263,7 @@ session:
   # The domain to protect.
   # Note: the authenticator must also be in that domain. If empty, the cookie
   # is restricted to the subdomain of the issuer.
-  domain: replaceme.org
+  domain: domain.org
 
   # The redis connection details
   redis:
@@ -342,7 +334,7 @@ notifier:
     host: smtp-server-hostname
     port: 25
     disable_require_tls: true
-    sender: replace@me.fr
+    sender: replace@domain.org
 
   # Sending an email using a Gmail account is as simple as the next section.
   # You need to create an app password by following: https://support.google.com/accounts/answer/185833?hl=en

examples/authelia_secrets.env

+AUTHELIA_AUTHENTICATION_BACKEND_LDAP_PASSWORD=xxxxxxx
+AUTHELIA_JWT_SECRET=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+AUTHELIA_SESSION_SECRET=xxxxxxxxxxxxxxxxx

examples/config_example.yml → examples/config.yml

@@ -3,12 +3,13 @@
 # "all" section contains variables used by several services at once or the paths to import sensitive files
 all:
       hostname: localhost  # Required. The hosting machine name
-      dashboard_port: 8001  # Required. The desired port (on the hosting machine) for the traefik container dashboard
-      http_port: 8888  # Required. The HTTP port docker will use on the hosting machine 
+      http_port: 8888  # Required. The HTTP port docker will use on the hosting machine
       https_port: 8889  # Required for Authelia. The HTTPS port docker will use on the hosting machine
-      proxy_ip: XXXXXXXXXXXX  # Required. IP of the upstream proxy (used by Traefik)
-      authentication_domain_name: XXXXXXXXXXXX  #  Required for Authelia. The authentication domain name.
+      proxy_ip: XXX.XXX.XXX.XXX  # Required. IP of the upstream proxy (used by Traefik)
+      authentication_domain_name: XXXXXXXX  #  Required for Authelia. The authentication domain name.
       authelia_config_path: /path/to/authelia_config.yml #  Required for Authelia. Path to the Authelia configuration file
+      authelia_secrets_env_path: /path/to/authelia/secrets.env #  Required for Authelia. Path to the env file containing passwords and secrets needed for Authelia
+      authelia-db_postgres_password: XXXXXXXX #  Required for Authelia.
 # galaxy-specific variables
 galaxy:
       galaxy_default_admin_email: gga@galaxy.org  # Required
@@ -17,11 +18,11 @@ galaxy:
       webapollo_user: admin_apollo@galaxy.org  # Required
       webapollo_password: apollopass  # Required
       galaxy_config_remote_user_maildomain: mydomain.com  # Required. The maildomain used by Galaxy authentication
-      persist_galaxy_data: "True"  # # Optional (default: True). If False, docker data will NOT be persisted on your host's file system and will be lost any time the galaxy container is recreated. Do not set this variable to "False" for production
+      persist_galaxy_data: "True"  # Optional (default: True). If False, docker data will NOT be persisted on your host's file system and will be lost any time the galaxy container is recreated. Do not set this variable to "False" for production
 # tripal-specific variables
 tripal:
       tripal_password: tripalpass  # Required. Tripal database password (also used by galaxy as an environment variable)
-      banner_path: /my/path/banner.png  # Optional. Custom banner path 
+      banner_path: /path/to/banner.png  # Optional. Use this to change the top banner in Tripal
       tripal_theme_name: tripal_gga   # Optional. Use this to use another theme
       tripal_theme_git_clone: http://gitlab.sb-roscoff.fr/abims/e-infra/tripal_gga.git  # Optional. Use this to install another theme.
 # jbrowse-specific variables

gga_init.py

@@ -61,14 +61,11 @@ class DeploySpeciesStack(speciesData.SpeciesData):
         # If the path specified is invalid (because it's empty or is still the default demo one),
         # use the default banner instead
         if "banner_path" in self.config.keys():
-            if self.config["banner_path"] != "/path/to/banner" or self.config["banner_path"] != "":
-                try:
-                    logging.debug("Custom banner path: %s" % self.config["banner_path"])
-                    if os.path.isfile(os.path.abspath(self.config["banner_path"])):
-                        shutil.copy(os.path.abspath(self.config["banner_path"]), "%s/banner.png" % self.species_dir)
-                except FileNotFoundError:
-                    logging.warning("Specified banner not found (%s), using default banner instead" % self.config["banner_path"])
-                    self.config.pop("banner_path", None)
+            if not config["banner_path"] == "" and os.path.isfile(os.path.abspath(config["banner_path"])):
+                banner_dest_path = os.path.abspath("./banner.png")
+                logging.info("Custom banner path: %s" % self.config["banner_path"])
+                if os.path.samefile(os.path.abspath(config["banner_path"]),banner_dest_path):
+                    shutil.copy(os.path.abspath(self.config["banner_path"]), banner_dest_path)
             else:
                 logging.debug("Using default banner for Tripal pages")
                 self.config.pop("banner_path", None)
@@ -81,13 +78,13 @@ class DeploySpeciesStack(speciesData.SpeciesData):
         try:
             shutil.copy(os.path.join(self.script_dir, "files/nginx_download.conf"), os.path.abspath("./nginx/conf/default.conf"))
         except Exception as exc:
-            logging.critical("Could not copy nginx configuration file for %s" % self.full_name)
+            logging.critical("Could not copy nginx configuration file for %s %s", self.genus, self.species)
             logging.critical(exc)            
 
         # Return to main directory
         os.chdir(self.main_dir)
 
-        logging.info("Directory tree generated for %s" % self.full_name)
+        logging.info("Directory tree generated for %s %s", self.genus, self.species)
 
 
     def make_compose_files(self):
@@ -119,20 +116,20 @@ class DeploySpeciesStack(speciesData.SpeciesData):
         render_vars = {**self.config, **input_vars}
 
         # Render the gspecies docker-compose file and write it
-        gspecies_compose_template = env.get_template("gspecies_compose_template.yml.j2")
+        gspecies_compose_template = env.get_template("gspecies_compose.yml.j2")
         gspecies_compose_output = gspecies_compose_template.render(render_vars)
         with open(os.path.join(self.species_dir, "docker-compose.yml"), "w") as gspecies_compose_file:
             logging.info("Writing %s docker-compose.yml" % self.genus_species)
             gspecies_compose_file.truncate(0)
             gspecies_compose_file.write(gspecies_compose_output)
 
-
-        galaxy_nginx_conf_template = env.get_template("galaxy_nginx.conf.j2")
-        galaxy_nginx_conf_output = galaxy_nginx_conf_template.render(render_vars)
-        with open(os.path.join(self.main_dir, "galaxy_nginx.conf"), "w") as galaxy_nginx_conf_file:
-            logging.debug("Writing the galaxy_nginx.conf file for %s" % self.genus_species)
-            galaxy_nginx_conf_file.truncate(0)
-            galaxy_nginx_conf_file.write(galaxy_nginx_conf_output)
+        if not os.path.isfile(os.path.join(self.main_dir, "galaxy_nginx.conf")):
+            galaxy_nginx_conf_template = env.get_template("galaxy_nginx.conf.j2")
+            galaxy_nginx_conf_output = galaxy_nginx_conf_template.render(render_vars)
+            with open(os.path.join(self.main_dir, "galaxy_nginx.conf"), "w") as galaxy_nginx_conf_file:
+                logging.debug("Writing the galaxy_nginx.conf file for %s" % self.genus_species)
+                galaxy_nginx_conf_file.truncate(0)
+                galaxy_nginx_conf_file.write(galaxy_nginx_conf_output)
 
         # Create the volumes (directory) of the species docker-compose file
         create_mounts(working_dir=".", main_dir=self.main_dir)
@@ -199,10 +196,10 @@ def make_traefik_compose_files(config, main_dir):
 
         # Jinja2 templating, handled using the python "jinja2" module
         file_loader = FileSystemLoader(script_dir + "/templates")
-        env = Environment(loader=file_loader)
+        env = Environment(loader=file_loader, trim_blocks=True, lstrip_blocks=True)
 
         if not os.path.isfile("./traefik/docker-compose.yml"):
-            traefik_compose_template = env.get_template("traefik_compose_template.yml.j2")
+            traefik_compose_template = env.get_template("traefik_compose.yml.j2")
             traefik_compose_output = traefik_compose_template.render(render_vars)
             with open(os.path.join(main_dir, "traefik/docker-compose.yml"), 'w') as traefik_compose_file:
                 logging.info("Writing traefik docker-compose.yml")
@@ -210,22 +207,12 @@ def make_traefik_compose_files(config, main_dir):
                 traefik_compose_file.write(traefik_compose_output)
 
         if "authelia_config_path" in config.keys():
-            if not config["authelia_config_path"] == "" or not config["authelia_config_path"] == "/path/to/authelia/config":
-                if os.path.isfile(os.path.abspath(config["authelia_config_path"])):
-                    try:
-                        shutil.copy(os.path.abspath(config["authelia_config_path"]), "./traefik/authelia/configuration.yml")
-                    except Exception as exc:
-                        logging.critical("Could not copy authelia configuration file")
-                        sys.exit(exc)
-                    # authelia_config_template = env.get_template(os.path.basename(config["authelia_config_path"]))
-                    # authelia_config_output = authelia_config_template.render(render_vars)
-                    # with open(os.path.join(main_dir, "traefik/authelia/configuration.yml"), 'w') as authelia_config_file:
-                    #     logging.info("Writing authelia configuration.yml")
-                    #     authelia_config_file.truncate(0)
-                    #     authelia_config_file.write(authelia_config_output)
-                else:
-                    logging.critical("Cannot find authelia configuration path (%s)" % config["authelia_config_path"])
-                    sys.exit()
+            if not config["authelia_config_path"] == "" and os.path.isfile(os.path.abspath(config["authelia_config_path"])):
+                try:
+                    shutil.copy(os.path.abspath(config["authelia_config_path"]), "./traefik/authelia/configuration.yml")
+                except Exception as exc:
+                    logging.critical("Could not copy authelia configuration file")
+                    sys.exit(exc)
             else:
                 logging.critical("Invalid authelia configuration path (%s)" % config["authelia_config_path"])
                 sys.exit()
@@ -341,10 +328,7 @@ def deploy_stacks(input_list, main_dir, deploy_traefik):
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Automatic data loading in containers and interaction "
-                                                 "with galaxy instances for GGA"
-                                                 ", following the protocol @ "
-                                                 "http://gitlab.sb-roscoff.fr/abims/e-infra/gga")
+    parser = argparse.ArgumentParser(description="Deploy GGA containers")
 
     parser.add_argument("input",
                         type=str,
@@ -373,11 +357,8 @@ if __name__ == "__main__":
     else:
         logging.basicConfig(level=logging.INFO)
 
-    # Parsing the config file if provided, using the default config otherwise
-    if not args.config:
-        args.config = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "config")
-    else:
-        args.config = os.path.abspath(args.config)
+    config_file = os.path.abspath(args.config)
+    config = utilities.parse_config(config_file)
 
     main_dir = None
     if not args.main_directory:
@@ -389,7 +370,6 @@ if __name__ == "__main__":
 
     # Create traefik directory and compose files if needed or specified
     if args.force_traefik or not os.path.isdir(os.path.join(os.path.abspath(main_dir), "traefik")):
-        config = utilities.parse_config(args.config)
         make_traefik_compose_files(config=config, main_dir=main_dir)
 
     unique_sp_dict_list = utilities.get_unique_species_dict_list(sp_dict_list=sp_dict_list)
@@ -407,31 +387,19 @@ if __name__ == "__main__":
                                                                      "/")
 
         # Parse the config yaml file
-        deploy_stack_for_current_organism.config = utilities.parse_config(args.config)
-
-        # Set the instance url attribute
-        for env_variable, value in deploy_stack_for_current_organism.config.items():
-            if env_variable == "hostname":
-                deploy_stack_for_current_organism.instance_url = value + \
-                                                                 deploy_stack_for_current_organism.genus_lowercase + \
-                                                                 "_" + deploy_stack_for_current_organism.species + \
-                                                                 "/galaxy/"
-                break
-            else:
-                deploy_stack_for_current_organism.instance_url = "http://localhost:8888/sp/{0}_{1}/galaxy/".format(
-                    deploy_stack_for_current_organism.genus_lowercase,
-                    deploy_stack_for_current_organism.species)
-                                                                    
+        deploy_stack_for_current_organism.config = config
+
         # Starting
-        logging.info("gga_init.py called for %s" % deploy_stack_for_current_organism.full_name)
+        logging.info("gga_init.py called for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
+        logging.debug("Jbrowse url to %s %s %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species, deploy_stack_for_current_organism.sex, deploy_stack_for_current_organism.strain)
 
         # Make/update directory tree
         deploy_stack_for_current_organism.make_directory_tree()
-        logging.info("Successfully generated the directory tree for %s" % deploy_stack_for_current_organism.full_name)
+        logging.info("Successfully generated the directory tree for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
 
         # Make compose files
         deploy_stack_for_current_organism.make_compose_files()
-        logging.info("Successfully generated the docker-compose files for %s" % deploy_stack_for_current_organism.full_name)
+        logging.info("Successfully generated the docker-compose files for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
 
     logging.info("Deploying stacks")
     if args.force_traefik:

serexec

@@ -4,11 +4,8 @@ set -e
 SERVICE_NAME=$1; shift
 
 TASK_ID=$(docker service ps --filter 'desired-state=running' $SERVICE_NAME -q)
-#we have only one node
-#NODE_ID=$(docker inspect --format '{{ .NodeID }}' $TASK_ID)
+#NODE_ID=$(docker inspect --format '{{ .NodeID }}' $TASK_ID) # if multiple nodes
 CONTAINER_ID=$(docker inspect --format '{{ .Status.ContainerStatus.ContainerID }}' $TASK_ID)
-#we have only one node
-#NODE_HOST=$(docker node inspect --format '{{ .Description.Hostname }}' $NODE_ID)
-#we have only one node
-#export DOCKER_HOST="ssh://$USER@$NODE_HOST"
+#NODE_HOST=$(docker node inspect --format '{{ .Description.Hostname }}' $NODE_ID) # if multiple nodes
+#export DOCKER_HOST="ssh://$USER@$NODE_HOST" # if multiple nodes
 docker exec -it $CONTAINER_ID "$@"

speciesData.py

@@ -15,14 +15,22 @@ class SpeciesData:
 
     """
 
+    def clean_string(self, string):
+        if not string is None and string != "":
+            clean_string = string.replace(" ", "_").replace("-", "_").replace("(", "").replace(")", "").replace("'", "").strip()
+            return clean_string
+        else:
+            return string
+
     def __init__(self, parameters_dictionary):
         # self.config_dictionary = None
         self.parameters_dictionary = parameters_dictionary
-        self.species = parameters_dictionary["description"]["species"].replace("(", "_").replace(")", "_").replace("-", "_")
-        self.genus = parameters_dictionary["description"]["genus"].replace("(", "_").replace(")", "_").replace("-", "_")
-        self.strain = parameters_dictionary["description"]["strain"].replace("(", "_").replace(")", "_").replace("-", "_")
-        self.sex = parameters_dictionary["description"]["sex"].replace("(", "_").replace(")", "_").replace("-", "_")
-        self.common = parameters_dictionary["description"]["common_name"].replace("(", "_").replace(")", "_").replace("-", "_")
+        parameters_dictionary_description=parameters_dictionary["description"]
+        self.species = self.clean_string(parameters_dictionary_description["species"])
+        self.genus = self.clean_string(parameters_dictionary_description["genus"])
+        self.strain = self.clean_string(parameters_dictionary_description["strain"])
+        self.sex = self.clean_string(parameters_dictionary_description["sex"])
+        self.common = self.clean_string(parameters_dictionary_description["common_name"])
 
         self.date = datetime.today().strftime("%Y-%m-%d")
         self.origin = parameters_dictionary["description"]["origin"]

templates/gspecies_compose_template.yml.j2 → templates/gspecies_compose.yml.j2

@@ -20,7 +20,7 @@ services:
             - "traefik.http.routers.{{ genus_species }}-nginx.middlewares=sp-auth,sp-app-trailslash,sp-prefix"
             {% else %}
             - "traefik.http.routers.{{ genus_species }}-nginx.entryPoints=web"
-            - "traefik.http.routers.{{ genus_species }}-nginx.middlewares=sp-app-trailslash,sp-prefix" # lg
+            - "traefik.http.routers.{{ genus_species }}-nginx.middlewares=sp-app-trailslash,sp-prefix"
             {% endif %}
             - "traefik.http.services.{{ genus_species }}-nginx.loadbalancer.server.port=80"
           restart_policy:
@@ -38,7 +38,7 @@ services:
             - ./docker_data/galaxy/:/export/:ro
             - ./src_data/:/project_data/:ro
             - ./src_data:/data:ro
-          {% if 'banner' in render_vars %}
+          {% if 'banner_path' is defined %}
             - ./banner.png:/var/www/html/banner.png:ro
           {% endif %}
             #- /groups/XXX/:/groups/XXX/:ro  # We do this when we have symlinks in src_data pointing to /groups/XXX/...
@@ -53,24 +53,10 @@ services:
             SITE_NAME: "{{ Genus_species }}"
             ELASTICSEARCH_HOST: elasticsearch.{{ genus_species }}
             ENABLE_JBROWSE: /jbrowse/?data=data/{{ genus_species_strain_sex }}
-            # This ENABLE_JBROWSE variable should point to the "best assembly" by default --> tag it in the input file and use it to define this variable correctly (also called
-            # unique id in the jbrowse tool parameters == both have to be identical)
-            {% if apollo == True %}
-            ENABLE_APOLLO: 1
-            {% else %}
-            ENABLE_APOLLO: 0
-            {% endif %}
-            {% if blast == True %}
-            ENABLE_BLAST: 1
-            {% else %}
-            ENABLE_BLAST: 0
-            {% endif %}
-            ENABLE_DOWNLOAD: 1
-            {% if wiki == True %}
-            ENABLE_WIKI: 1
-            {% else %}
-            ENABLE_WIKI: 0
-            {% endif %}
+            ENABLE_APOLLO: {{ apollo }}
+            ENABLE_BLAST: {{ blast }}
+            ENABLE_DOWNLOAD: {{ download }}
+            ENABLE_WIKI: {{ wiki }}
             ENABLE_GO: 0
             ENABLE_ORTHOLOGY: 0
             ENABLE_ORTHOLOGY_LINKS: 0
@@ -89,7 +75,7 @@ services:
             - "traefik.http.routers.{{ genus_species }}-tripal.middlewares=sp-auth,sp-trailslash,sp-prefix,tripal-addprefix"
             {% else %}
             - "traefik.http.routers.{{ genus_species }}-tripal.entryPoints=web"
-            - "traefik.http.routers.{{ genus_species }}-tripal.middlewares=sp-trailslash,sp-prefix,tripal-addprefix" # lg
+            - "traefik.http.routers.{{ genus_species }}-tripal.middlewares=sp-trailslash,sp-prefix,tripal-addprefix"
             {% endif %}
             - "traefik.http.services.{{ genus_species }}-tripal.loadbalancer.server.port=80"
           restart_policy:
@@ -112,14 +98,9 @@ services:
 
     elasticsearch:
         image: docker.elastic.co/elasticsearch/elasticsearch:6.6.1
-        #deploy:
-          #resources:
-            #limits:
-              #memory: 500M
         volumes:
             - ./docker_data/elastic_search_index/:/usr/share/elasticsearch/data/
         environment:
-            # bootstrap.memory_lock: "true"
             xpack.security.enabled: "false"
             xpack.monitoring.enabled: "false"
             xpack.ml.enabled: "false"
@@ -134,15 +115,11 @@ services:
     galaxy:
         image: quay.io/galaxy-genome-annotation/docker-galaxy-annotation:gmod
         volumes:
-            {% if persist_galaxy_data is defined %}
-            {% if persist_galaxy_data == "False" %}
+            {% if (persist_galaxy_data is defined) and (persist_galaxy_data == "False") %}
             #- ./docker_data/galaxy/:/export/
             {% else %}
             - ./docker_data/galaxy/:/export/
             {% endif %}
-            {% else %}
-            - ./docker_data/galaxy/:/export/
-            {% endif %}
             - ./src_data/:/project_data/:ro
             #- /groups/XXX/:/groups/XXX/:ro  # We do this when we have symlinks in src_data pointing to /groups/XXX/...
             - ./docker_data/jbrowse/:/jbrowse/data/
@@ -158,7 +135,7 @@ services:
             GALAXY_DEFAULT_ADMIN_EMAIL: "{{ galaxy_default_admin_email }}"
             GALAXY_DEFAULT_ADMIN_USER: "{{ galaxy_defaut_admin_user }}"
             GALAXY_DEFAULT_ADMIN_PASSWORD: "{{ galaxy_default_admin_password }}"
-            GALAXY_CONFIG_ADMIN_USERS: "admin@galaxy.org,{{ galaxy_default_admin_email }}"   # admin@galaxy.org is the default (leave it), gogepp@bipaa is a shared ldap user we use to connect
+            GALAXY_CONFIG_ADMIN_USERS: "admin@galaxy.org,{{ galaxy_default_admin_email }}"   # admin@galaxy.org is the default (leave it), galaxy_default_admin_email is a shared ldap user we use to connect
             ENABLE_FIX_PERMS: 0
             PROXY_PREFIX: /sp/{{ genus_species }}/galaxy
             GALAXY_TRIPAL_URL: http://tripal.{{ genus_species }}/tripal/
@@ -186,12 +163,12 @@ services:
             - "traefik.http.routers.{{ genus_species }}-galaxy.middlewares=sp-auth,sp-app-trailslash,sp-app-prefix"
             {% else %}
             - "traefik.http.routers.{{ genus_species }}-galaxy.entryPoints=web"
-            - "traefik.http.routers.{{ genus_species }}-galaxy.middlewares=sp-app-trailslash,sp-app-prefix" #lg
+            - "traefik.http.routers.{{ genus_species }}-galaxy.middlewares=sp-app-trailslash,sp-app-prefix"
             {% endif %}
             - "traefik.http.services.{{ genus_species }}-galaxy.loadbalancer.server.port=80"
             - "traefik.http.routers.{{ genus_species }}-gga_load-galaxy.rule=(Host(`localhost`) && PathPrefix(`/sp/{{ genus_species }}/galaxy`))"
             - "traefik.http.routers.{{ genus_species }}-gga_load-galaxy.entryPoints=web"
-
+            - "traefik.http.routers.{{ genus_species }}-gga_load-galaxy.middlewares=sp-app-trailslash,sp-app-prefix"
           restart_policy:
             condition: on-failure
             delay: 5s
@@ -217,7 +194,7 @@ services:
             - "traefik.http.routers.{{ genus_species }}-jbrowse.middlewares=sp-auth,sp-app-trailslash,sp-app-prefix"
             {% else %}
             - "traefik.http.routers.{{ genus_species }}-jbrowse.entryPoints=web"
-            - "traefik.http.routers.{{ genus_species }}-jbrowse.middlewares=sp-app-trailslash,sp-app-prefix" #lg
+            - "traefik.http.routers.{{ genus_species }}-jbrowse.middlewares=sp-app-trailslash,sp-app-prefix"
             {% endif %}
             - "traefik.http.services.{{ genus_species }}-jbrowse.loadbalancer.server.port=80"
           restart_policy:
@@ -226,7 +203,7 @@ services:
             max_attempts: 3
             window: 120s
 
-    {% if blast == True %}
+    {% if blast is defined and blast == 1 %}
     blast:
         image: quay.io/abretaud/sf-blast:latest
         depends_on:
@@ -236,7 +213,7 @@ services:
             UPLOAD_LIMIT: 20M
             MEMORY_LIMIT: 128M
             DB_NAME: 'postgres'
-            ADMIN_EMAIL: 'g.ga@sb-roscoff.fr'  # email sender
+            ADMIN_EMAIL: 'g.ga@domain.org'  # email sender
             ADMIN_NAME: 'gga'  # email sender name
             JOBS_METHOD: 'local'   # Can be local (= no sge jobs, but run inside the container) or drmaa (= to submit to a cluster)
             JOBS_WORK_DIR: '/tmp/blast_jobs/'  # disk accessible both from compute nodes and mounted in this docker (at the same path)
@@ -252,8 +229,8 @@ services:
             #JOBS_DRMAA_NATIVE: '-p web' # This line and following for slurm
             #DRMAA_METHOD: 'slurm' # This line and following for slurm
         volumes:
-            - ../blast-themes/abims/:/var/www/blast/app/Resources/:ro # You can theme the app
-            - /usr/local/genome2/:/usr/local/genome2/:ro # path for blast executables
+            - ../blast-themes/my_theme/:/var/www/blast/app/Resources/:ro # You can theme the app
+            - /path/to/blast/exe/:/path/to/blast/exe/:ro # path for blast executables
             - /db/:/db/:ro # for access to indexed blast databases
             #- /data1/sge/:/usr/local/sge/:ro # an sge install
             #- /xxxx/blast_jobs/:/xxxx/blast_jobs/ # (for drmaa mode only)
@@ -277,7 +254,7 @@ services:
             - "traefik.http.routers.{{ genus_species }}-blast.middlewares=sp-big-req,sp-auth,sp-app-trailslash,sp-app-prefix"
             {% else %}
             - "traefik.http.routers.{{ genus_species }}-blast.entryPoints=web"
-            - "traefik.http.routers.{{ genus_species }}-blast.middlewares=sp-big-req,sp-app-trailslash,sp-app-prefix" # lg
+            - "traefik.http.routers.{{ genus_species }}-blast.middlewares=sp-big-req,sp-app-trailslash,sp-app-prefix"
             {% endif %}
             - "traefik.http.services.{{ genus_species }}-blast.loadbalancer.server.port=80"
           restart_policy:
@@ -297,49 +274,9 @@ services:
             - {{ genus_species }}
     {% endif %}
 
-    {% if wiki == True %}
-    wiki:
-        image: quay.io/abretaud/mediawiki
-        environment:
-            MEDIAWIKI_SERVER: http://localhost
-            MEDIAWIKI_PROXY_PREFIX: /sp/{{ genus_species }}/wiki
-            MEDIAWIKI_SITENAME: {{ Genus }} {{ species }}
-            MEDIAWIKI_SECRET_KEY: XXXXXXXXXX
-            MEDIAWIKI_DB_HOST: wiki-db.{{genus_species }}
-            MEDIAWIKI_DB_PASSWORD: password
-            MEDIAWIKI_ADMIN_USER: abretaud   # ldap user
-        depends_on:
-            - wiki-db
-        volumes:
-            - ./docker_data/wiki_uploads:/images
-            #- ../bipaa_wiki.png:/var/www/mediawiki/resources/assets/wiki.png:ro # To change the logo at the top left
-        networks:
-            - traefikbig
-            - {{ genus_species }}
-        deploy:
-          labels:
-            - "traefik.http.routers.{{ genus_species }}-wiki.rule=(Host(`{{ hostname }}`) && PathPrefix(`/sp/{{ genus_species }}/wiki`))"
-            - "traefik.http.routers.{{ genus_species }}-wiki.tls=true"
-            - "traefik.http.routers.{{ genus_species }}-wiki.entryPoints={{ entrypoint }}"
-            - "traefik.http.routers.{{ genus_species }}-wiki.middlewares=sp-big-req,sp-auth,sp-app-trailslash,sp-app-prefix"
-            - "traefik.http.services.{{ genus_species }}-wiki.loadbalancer.server.port=80"
-          restart_policy:
-            condition: on-failure
-            delay: 5s
-            max_attempts: 3
-            window: 120s
-
-    wiki-db:
-        image: postgres:9.6-alpine
-        volumes:
-            - ./docker_data/wiki_db/:/var/lib/postgresql/data/
-        networks:
-            - {{ genus_species }}
-    {% endif %}
-    
 networks:
     traefikbig:
         external: true
     {{ genus_species }}:
         driver: overlay
-        name: {{ genus_species }}
+        name: {{ genus_species }}
\ No newline at end of file

templates/organisms.yml.j2

+{{ name }}:
+  description:
+    genus: {{ genus }}
+    species: {{ species }}
+    sex: {{ sex }}
+    strain: {{ strain }}
+    common_name: {{ common_name }}
+    origin: {{ origin }}
+    {% if main_species is defined and main_species is sameas true %}
+    main_species: yes
+    {% endif %}
+  data:
+    genome_path: {{ genome_path }}
+    transcripts_path: {{ transcripts_path }}
+    proteins_path: {{ proteins_path }}
+    gff_path: {{ gff_path }} # Mandatory!
+    interpro_path: {{ interpro_path }}
+    orthofinder_path: {{ orthofinder_path }}
+    blastp_path: {{ blastp_path }}
+    blastx_path: {{ blastx_path }}
+    genome_version: {{ genome_version }}
+    ogs_version: {{ ogs_version }}
+    performed_by: {{ performed_by }}
+  services:
+    blast: {{ blast }}
\ No newline at end of file

templates/traefik_compose_template.yml.j2 → templates/traefik_compose.yml.j2

@@ -5,19 +5,15 @@ services:
     command:
       - "--api"
       - "--api.dashboard"
-#      - "--api.insecure=true" # added by lg to debug, for dashboard
       - "--log.level=DEBUG"
       - "--providers.docker"
       - "--providers.docker.swarmMode=true"
-      - "--providers.docker.network=traefikbig" # changed by lg from traefik to traefikbig
+      - "--providers.docker.network=traefikbig"
       - "--entryPoints.web.address=:80"
-      - "--entryPoints.web.forwardedHeaders.trustedIPs={{ proxy_ip }}"  # The ips of our upstream proxies: eci
+      - "--entryPoints.web.forwardedHeaders.trustedIPs={{ proxy_ip }}"  # The ips of our upstream proxies
       - "--entryPoints.webs.address=:443"
-      - "--entryPoints.webs.forwardedHeaders.trustedIPs={{ proxy_ip }}"  # The ips of our upstream proxies: eci
+      - "--entryPoints.webs.forwardedHeaders.trustedIPs={{ proxy_ip }}"  # The ips of our upstream proxies
     ports:
-      {% if dashboard_port is defined %}
-      - {{ dashboard_port }}:8080 # added by lg to debug, for dashboard
-      {% endif %}
       - {{ http_port }}:80
       {% if https_port is defined %}
       - {{ https_port }}:443
@@ -31,37 +27,39 @@ services:
         constraints:
           - node.role == manager
       labels:
-#        - "traefik.http.routers.traefik-api.rule=PathPrefix(`/traefik`)"
-        - "traefik.http.routers.traefik-api.rule=PathPrefix(`/api`) || PathPrefix(`/dashboard`) || PathPrefix(`/traefik`)" # lg
+        - "traefik.http.routers.traefik-api.rule=PathPrefix(`/api`) || PathPrefix(`/dashboard`) || PathPrefix(`/traefik`)"
         {% if https_port is defined %}
         - "traefik.http.routers.traefik-api.tls=true"
         - "traefik.http.routers.traefik-api.entryPoints=webs"
         {% else %}
-        - "traefik.http.routers.traefik-api.entryPoints=web" # lg
+        - "traefik.http.routers.traefik-api.entryPoints=web"
         {% endif %}
         - "traefik.http.routers.traefik-api.service=api@internal"
         - "traefik.http.middlewares.traefik-strip.stripprefix.prefixes=/traefik"
-        - "traefik.http.middlewares.traefik-auth.forwardauth.address=http://authelia:9091/api/verify?rd=https://auth.abims-gga.sb-roscoff.fr/"
+        - "traefik.http.middlewares.traefik-auth.forwardauth.address=http://authelia:9091/api/verify?rd=https://{{ authentication_domain_name }}/"
         - "traefik.http.middlewares.traefik-auth.forwardauth.trustForwardHeader=true"
-#        - "traefik.http.routers.traefik-api.middlewares=traefik-auth,traefik-strip"
-        - "traefik.http.routers.traefik-api.middlewares=traefik-strip" # lg
+        {% if https_port is defined %}
+        - "traefik.http.routers.traefik-api.middlewares=traefik-auth,traefik-strip"
+        {% else %}
+        - "traefik.http.routers.traefik-api.middlewares=traefik-strip"
+        {% endif %}
         # Dummy service for Swarm port detection. The port can be any valid integer value.
         - "traefik.http.services.traefik-svc.loadbalancer.server.port=9999"
         # Some generally useful middlewares for organisms hosting
-        - "traefik.http.middlewares.sp-auth.forwardauth.address=http://authelia:9091/api/verify?rd=https://auth.abims-gga.sb-roscoff.fr/"
+        - "traefik.http.middlewares.sp-auth.forwardauth.address=http://authelia:9091/api/verify?rd=https://{{ authentication_domain_name }}/"
         - "traefik.http.middlewares.sp-auth.forwardauth.trustForwardHeader=true"
         - "traefik.http.middlewares.sp-auth.forwardauth.authResponseHeaders=Remote-User,Remote-Groups"
         {% if https_port is defined %}
         - "traefik.http.middlewares.sp-trailslash.redirectregex.regex=^(https?://[^/]+/sp/[^/]+)$$"
         {% else %}
-        - "traefik.http.middlewares.sp-trailslash.redirectregex.regex=^(http?://[^/]+/sp/[^/]+)$$" # lg
+        - "traefik.http.middlewares.sp-trailslash.redirectregex.regex=^(http?://[^/]+/sp/[^/]+)$$"
         {% endif %}
         - "traefik.http.middlewares.sp-trailslash.redirectregex.replacement=$${1}/"
         - "traefik.http.middlewares.sp-trailslash.redirectregex.permanent=true"
         {% if https_port is defined %}
         - "traefik.http.middlewares.sp-app-trailslash.redirectregex.regex=^(https?://[^/]+/sp/[^/]+/[^/]+)$$"
         {% else %}
-        - "traefik.http.middlewares.sp-app-trailslash.redirectregex.regex=^(http?://[^/]+/sp/[^/]+/[^/]+)$$" # lg
+        - "traefik.http.middlewares.sp-app-trailslash.redirectregex.regex=^(http?://[^/]+/sp/[^/]+/[^/]+)$$"
         {% endif %}
         - "traefik.http.middlewares.sp-app-trailslash.redirectregex.replacement=$${1}/"
         - "traefik.http.middlewares.sp-app-trailslash.redirectregex.permanent=true"
@@ -86,6 +84,10 @@ services:
         - authelia-db
     volumes:
       - ./authelia/:/etc/authelia/:ro
+    {% if authelia_secrets_env_path is defined %}
+    env_file:
+      - ./authelia/secrets.env
+    {% endif %}
     deploy:
       labels:
         - "traefik.http.routers.authelia.rule=Host(`{{ authentication_domain_name }}`)"
@@ -113,7 +115,7 @@ services:
   authelia-db:
       image: postgres:12.2-alpine
       environment:
-          POSTGRES_PASSWORD: z3A,hQ-9
+          POSTGRES_PASSWORD: {{ authelia-db_postgres_password }}
       volumes:
           - ./docker_data/authelia_db/:/var/lib/postgresql/data/
       networks:
@@ -132,4 +134,4 @@ networks:
     name: traefikbig
     ipam:
       config:
-        - subnet: 10.50.0.0/16
+        - subnet: 10.50.0.0/16
\ No newline at end of file

utilities.py

@@ -123,6 +123,16 @@ def get_species_history_id(instance, full_name):
 
     return [history_id, show_history]
 
+def get_gspecies_string_from_sp_dict(sp_dict):
+
+    gspecies = ""
+    for k, v in sp_dict.items():
+        for k2, v2 in v.items():
+            if k2 == "genus":
+                gspecies = gspecies.lower() + v2
+            elif k2 == "species":
+                gspecies = gspecies.lower() + "_" + v2
+    return gspecies
 
 def get_unique_species_str_list(sp_dict_list):
     """
@@ -137,15 +147,9 @@ def get_unique_species_str_list(sp_dict_list):
     unique_species_li = []
 
     for sp in sp_dict_list:
-        for k, v in sp.items():
-            sp_gspecies = ""
-            for k2, v2 in v.items():
-                if k2 == "genus":
-                    sp_gspecies = sp_gspecies.lower() + v2
-                elif k2 == "species":
-                    sp_gspecies = sp_gspecies.lower() + "_" + v2
-            if sp_gspecies not in unique_species_li and sp_gspecies != "":
-                unique_species_li.append(sp_gspecies)
+        sp_gspecies = get_gspecies_string_from_sp_dict(sp)
+        if sp_gspecies not in unique_species_li and sp_gspecies != "":
+            unique_species_li.append(sp_gspecies)
                 
     return unique_species_li
 
@@ -165,11 +169,11 @@ def get_unique_species_dict_list(sp_dict_list):
     unique_species_genus_species = get_unique_species_str_list(sp_dict_list=sp_dict_list)
 
     for sp in sp_dict_list:
-        for gspecies in unique_species_genus_species:
-            if gspecies not in unique_species_dict.keys():
-                unique_species_dict[gspecies] = sp
-            else:
-                continue
+        gspecies = get_gspecies_string_from_sp_dict(sp)
+        if gspecies not in unique_species_dict.keys() or sp["description"]["main_species"] == True :
+            unique_species_dict[gspecies] = sp
+        else:
+            continue
 
     for k, v in unique_species_dict.items():
         unique_species_list_of_dict.append(v)