Merge branch 'corrections_cloud' into 'dev'

Corrections cloud

See merge request !2

examples/config_example.yml

@@ -2,28 +2,28 @@
 
 # "all" section contains variables used by several services at once or the paths to import sensitive files
 all:
-      hostname: localhost  # The hosting machine name
-      dashboard_port: 8001  # The desired port (on the hosting machine) for the traefik container dashboard
-      http_port: 8888  # The HTTP port docker will use on the hosting machine 
-      https_port: 8889  # The HTTPS port docker will use on the hosting machine. Required for Authelia
-      proxy_ip: XXXXXXXXXXXX  # IP of the upstream proxy (used by Traefik)
-      auth_hostname: XXXXXXXXXXXX  #  Required for Authelia. The authentication domain name.
+      hostname: localhost  # Required. The hosting machine name
+      dashboard_port: 8001  # Required. The desired port (on the hosting machine) for the traefik container dashboard
+      http_port: 8888  # Required. The HTTP port docker will use on the hosting machine 
+      https_port: 8889  # Required for Authelia. The HTTPS port docker will use on the hosting machine
+      proxy_ip: XXXXXXXXXXXX  # Required. IP of the upstream proxy (used by Traefik)
+      authentication_domain_name: XXXXXXXXXXXX  #  Required for Authelia. The authentication domain name.
       authelia_config_path: /path/to/authelia_config.yml #  Required for Authelia. Path to the Authelia configuration file
 # galaxy-specific variables
 galaxy:
-      galaxy_default_admin_email: gga@galaxy.org
-      galaxy_defaut_admin_user: gga
-      galaxy_default_admin_password: password
-      webapollo_user: admin_apollo@galaxy.org
-      webapollo_password: apollopass
-      galaxy_config_remote_user_maildomain: mydomain.com  # The maildomain used by Galaxy authentication
-      persist_galaxy_data: "True"  # Optional. If True, docker data will be backed up. Do not set this variable to "false" for production
+      galaxy_default_admin_email: gga@galaxy.org  # Required
+      galaxy_defaut_admin_user: gga  # Required
+      galaxy_default_admin_password: password  # Required
+      webapollo_user: admin_apollo@galaxy.org  # Required
+      webapollo_password: apollopass  # Required
+      galaxy_config_remote_user_maildomain: mydomain.com  # Required. The maildomain used by Galaxy authentication
+      persist_galaxy_data: "True"  # # Optional (default: True). If False, docker data will NOT be persisted on your host's file system and will be lost any time the galaxy container is recreated. Do not set this variable to "False" for production
 # tripal-specific variables
 tripal:
-      tripal_password: tripalpass  # Tripal database password (also used by galaxy as an environment variable)
+      tripal_password: tripalpass  # Required. Tripal database password (also used by galaxy as an environment variable)
       banner_path: /my/path/banner.png  # Optional. Custom banner path 
       tripal_theme_name: tripal_gga   # Optional. Use this to use another theme
       tripal_theme_git_clone: http://gitlab.sb-roscoff.fr/abims/e-infra/tripal_gga.git  # Optional. Use this to install another theme.
 # jbrowse-specific variables
 jbrowse:
-      menu_url: "http://localhost:8888/" # Used with run_workflow_phaeoexplorer.py. If present, this variable is used to define jbrowse menu_url, if absent, will use https://hostname:https_port instead
\ No newline at end of file
+      menu_url: "http://localhost:8888/" # Optional. Used with run_workflow_phaeoexplorer.py: if present, this variable is used to define JBrowse menu_url (to define the template url for the JBrowse feature's link to Tripal), if absent, will use default "https://hostname"
\ No newline at end of file

gga_init.py

@@ -79,11 +79,10 @@ class DeploySpeciesStack(speciesData.SpeciesData):
         # Create nginx dirs and write/re-write nginx conf
         make_dirs(dir_paths_li=["./nginx", "./nginx/conf"])
         try:
-            with open(os.path.abspath("./nginx/conf/default.conf"), 'w') as conf:
-                conf.write("server {\n\tlisten 80;\n\tserver_name ~.;\n\tlocation /download/ {\n\t\talias /project_data/; \n\t\tautoindex on;\n\t}\n}")  # The species nginx conf
-        except OSError as exc:
-            logging.critical("Cannot edit NginX conf file")
-            sys.exit(exc)
+            shutil.copy(os.path.join(self.script_dir, "files/nginx_download.conf"), os.path.abspath("./nginx/conf/default.conf"))
+        except Exception as exc:
+            logging.critical("Could not copy nginx configuration file for %s" % self.full_name)
+            logging.critical(exc)            
 
         # Return to main directory
         os.chdir(self.main_dir)
@@ -107,7 +106,7 @@ class DeploySpeciesStack(speciesData.SpeciesData):
 
         # Jinja2 templating, handled using the python "jinja2" module
         file_loader = FileSystemLoader(self.script_dir + "/templates")
-        env = Environment(loader=file_loader)
+        env = Environment(loader=file_loader, trim_blocks=True, lstrip_blocks=True)
 
         # We need a dict holding all key (variables) - values that needs to be replaced in the template as our rendering dict
         # To do so we need both input file vars and config vars
@@ -127,6 +126,14 @@ class DeploySpeciesStack(speciesData.SpeciesData):
             gspecies_compose_file.truncate(0)
             gspecies_compose_file.write(gspecies_compose_output)
 
+
+        galaxy_nginx_conf_template = env.get_template("galaxy_nginx.conf.j2")
+        galaxy_nginx_conf_output = galaxy_nginx_conf_template.render(render_vars)
+        with open(os.path.join(self.main_dir, "galaxy_nginx.conf"), "w") as galaxy_nginx_conf_file:
+            logging.debug("Writing the galaxy_nginx.conf file for %s" % self.genus_species)
+            galaxy_nginx_conf_file.truncate(0)
+            galaxy_nginx_conf_file.write(galaxy_nginx_conf_output)
+
         # Create the volumes (directory) of the species docker-compose file
         create_mounts(working_dir=".", main_dir=self.main_dir)
 
@@ -202,7 +209,7 @@ def make_traefik_compose_files(config, main_dir):
                 traefik_compose_file.truncate(0)
                 traefik_compose_file.write(traefik_compose_output)
 
-        if config["authelia_config_path"]:
+        if "authelia_config_path" in config.keys():
             if not config["authelia_config_path"] == "" or not config["authelia_config_path"] == "/path/to/authelia/config":
                 if os.path.isfile(os.path.abspath(config["authelia_config_path"])):
                     try:
@@ -217,11 +224,14 @@ def make_traefik_compose_files(config, main_dir):
                     #     authelia_config_file.truncate(0)
                     #     authelia_config_file.write(authelia_config_output)
                 else:
-                    logging.critical("Cannot find authelia configuration template path (%s)" % config["authelia_config_path"])
+                    logging.critical("Cannot find authelia configuration path (%s)" % config["authelia_config_path"])
                     sys.exit()
+            else:
+                logging.critical("Invalid authelia configuration path (%s)" % config["authelia_config_path"])
+                sys.exit()
 
         # Path to the authelia users in the repo
-        authelia_users_path = script_dir + "/templates/authelia_users_template.yml"
+        authelia_users_path = script_dir + "/files/authelia_users_template.yml"
         # Copy authelia "users" file
         if not os.path.isfile("./traefik/authelia/users.yml"):
             shutil.copy(authelia_users_path, "./traefik/authelia/users.yml")

run_workflow_phaeoexplorer.py

@@ -492,9 +492,12 @@ class RunWorkflow(speciesData.SpeciesData):
             # In case of the Jbrowse workflow, we unfortunately have to manually edit the parameters instead of setting them
             # as runtime values, using runtime parameters makes the tool throw an internal critical error ("replace not found" error)
             # Scratchgmod test: need "http" (or "https"), the hostname (+ port)
-            # TODO --> use menu_url variable in config to set up the menu url
+            if "menu_url" not in self.config.keys():
+                jbrowse_menu_url = "https://{hostname}/sp/{genus_sp}/feature/{Genus}/{species}/mRNA/{id}".format(hostname=self.config["hostname"], genus_sp=self.genus_species, Genus=self.genus_uppercase, species=self.species, id="{id}")
+            else:
+                jbrowse_menu_url = self.config["menu_url"]
             if workflow_name == "Jbrowse":
-                workflow_dict["steps"]["2"]["tool_state"] = workflow_dict["steps"]["2"]["tool_state"].replace("__MENU_URL__", "https://{hostname}/sp/{genus_sp}/feature/{Genus}/{species}/{id}".format(hostname=self.config["hostname"], genus_sp=self.genus_species, Genus=self.genus_uppercase, species=self.species, id="{id}"))
+                workflow_dict["steps"]["2"]["tool_state"] = workflow_dict["steps"]["2"]["tool_state"].replace("__MENU_URL__", jbrowse_menu_url)
                 # The UNIQUE_ID is specific to a combination genus_species_strain_sex so every combination should have its unique workflow
                 # in galaxy --> define a naming method for these workflows
                 workflow_dict["steps"]["3"]["tool_state"] = workflow_dict["steps"]["3"]["tool_state"].replace("__FULL_NAME__", self.full_name).replace("__UNIQUE_ID__", self.species_folder_name)

speciesData.py

@@ -80,6 +80,7 @@ class SpeciesData:
         self.datasets = dict()
         self.config = None  # Custom config used to set environment variables inside containers
         self.species_folder_name = "_".join(utilities.filter_empty_not_empty_items([self.genus_lowercase.lower(), self.species.lower(), self.strain.lower(), self.sex.lower()])["not_empty"])
+        self.species_folder_name  = self.species_folder_name .replace("-", "_")
         self.existing_folders_cache = {}
         self.bam_metadata_cache = {}
 

templates/galaxy_nginx.conf.j2

+uwsgi_param  QUERY_STRING       $query_string;
+uwsgi_param  REQUEST_METHOD     $request_method;
+uwsgi_param  CONTENT_TYPE       $content_type;
+uwsgi_param  CONTENT_LENGTH     $content_length;
+
+uwsgi_param  REQUEST_URI        $request_uri;
+uwsgi_param  PATH_INFO          $document_uri;
+uwsgi_param  DOCUMENT_ROOT      $document_root;
+uwsgi_param  SERVER_PROTOCOL    $server_protocol;
+uwsgi_param  REQUEST_SCHEME     $scheme;
+uwsgi_param  HTTPS              $https if_not_empty;
+
+uwsgi_param  REMOTE_ADDR        $remote_addr;
+uwsgi_param  REMOTE_PORT        $remote_port;
+uwsgi_param  SERVER_PORT        $server_port;
+uwsgi_param  SERVER_NAME        $server_name;
+
+uwsgi_param HTTP_REMOTE_USER {% if https_port is defined %}$http_remote_user if_not_empty;{% elif galaxy_defaut_admin_user is defined %}{{ galaxy_defaut_admin_user }};{% else %}$http_remote_user if_not_empty;{% endif %}

templates/gspecies_compose_template.yml.j2

-#jinja2: lstrip_blocks: “True”, trim_blocks: "True"
 # ./docker_data is created and filled with persistent data that should be backuped
 version: '3.7'
 services:
@@ -135,7 +134,6 @@ services:
     galaxy:
         image: quay.io/galaxy-genome-annotation/docker-galaxy-annotation:gmod
         volumes:
-            - ../galaxy_data_libs_SI.py:/opt/galaxy_data_libs_SI.py
             {% if persist_galaxy_data is defined %}
             {% if persist_galaxy_data == "False" %}
             #- ./docker_data/galaxy/:/export/

templates/traefik.yml

-version: '3.7'
-services:
-  traefik:
-    image: traefik:2.1.6
-    command:
-      - "--api"
-      - "--api.dashboard"
-#      - "--api.insecure=true" # added by lg to debug, for dashboard
-      - "--log.level=DEBUG"
-      - "--providers.docker"
-      - "--providers.docker.swarmMode=true"
-      - "--providers.docker.network=traefikbig" # changed by lg from traefik to traefikbig
-      - "--entryPoints.web.address=:80"
-      - "--entryPoints.web.forwardedHeaders.trustedIPs=192.168.1.133"  # The ips of our upstream proxies: eci
-      - "--entryPoints.webs.address=:443"
-      - "--entryPoints.webs.forwardedHeaders.trustedIPs=192.168.1.133"  # The ips of our upstream proxies: eci
-    ports:
-      - 8001:8080 # added by lg to debug, for dashboard
-      - 8888:80
-      - 8889:443
-    networks:
-      - traefikbig
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock
-    deploy:
-      placement:
-        constraints:
-          - node.role == manager
-      labels:
-#        - "traefik.http.routers.traefik-api.rule=PathPrefix(`/traefik`)"
-        - "traefik.http.routers.traefik-api.rule=PathPrefix(`/api`) || PathPrefix(`/dashboard`) || PathPrefix(`/traefik`)" # lg
-#        - "traefik.http.routers.traefik-api.tls=true"
-        - "traefik.http.routers.traefik-api.entryPoints=web" # lg
-#        - "traefik.http.routers.traefik-api.entryPoints=webs"
-        - "traefik.http.routers.traefik-api.service=api@internal"
-        - "traefik.http.middlewares.traefik-strip.stripprefix.prefixes=/traefik"
-        - "traefik.http.middlewares.traefik-auth.forwardauth.address=http://authelia:9091/api/verify?rd=https://auth.abims-gga.sb-roscoff.fr/"
-        - "traefik.http.middlewares.traefik-auth.forwardauth.trustForwardHeader=true"
-#        - "traefik.http.routers.traefik-api.middlewares=traefik-auth,traefik-strip"
-        - "traefik.http.routers.traefik-api.middlewares=traefik-strip" # lg
-        # Dummy service for Swarm port detection. The port can be any valid integer value.
-        - "traefik.http.services.traefik-svc.loadbalancer.server.port=9999"
-        # Some generally useful middlewares for organisms hosting
-        - "traefik.http.middlewares.sp-auth.forwardauth.address=http://authelia:9091/api/verify?rd=https://auth.abims-gga.sb-roscoff.fr/"
-        - "traefik.http.middlewares.sp-auth.forwardauth.trustForwardHeader=true"
-        - "traefik.http.middlewares.sp-auth.forwardauth.authResponseHeaders=Remote-User,Remote-Groups"
-#        - "traefik.http.middlewares.sp-trailslash.redirectregex.regex=^(https?://[^/]+/sp/[^/]+)$$"
-        - "traefik.http.middlewares.sp-trailslash.redirectregex.regex=^(http?://[^/]+/sp/[^/]+)$$" # lg
-        - "traefik.http.middlewares.sp-trailslash.redirectregex.replacement=$${1}/"
-        - "traefik.http.middlewares.sp-trailslash.redirectregex.permanent=true"
-#        - "traefik.http.middlewares.sp-app-trailslash.redirectregex.regex=^(https?://[^/]+/sp/[^/]+/[^/]+)$$"
-        - "traefik.http.middlewares.sp-app-trailslash.redirectregex.regex=^(http?://[^/]+/sp/[^/]+/[^/]+)$$" # lg
-        - "traefik.http.middlewares.sp-app-trailslash.redirectregex.replacement=$${1}/"
-        - "traefik.http.middlewares.sp-app-trailslash.redirectregex.permanent=true"
-        - "traefik.http.middlewares.sp-prefix.stripprefixregex.regex=/sp/[^/]+"
-        - "traefik.http.middlewares.sp-app-prefix.stripprefixregex.regex=/sp/[^/]+/[^/]+"
-        - "traefik.http.middlewares.tripal-addprefix.addprefix.prefix=/tripal"
-        - "traefik.http.middlewares.sp-big-req.buffering.maxRequestBodyBytes=50000000"
-        - "traefik.http.middlewares.sp-huge-req.buffering.maxRequestBodyBytes=2000000000"
-      restart_policy:
-        condition: on-failure
-        delay: 5s
-        max_attempts: 3
-        window: 120s
-
-  authelia:
-    image: authelia/authelia:4.12.0
-    networks:
-      - traefikbig
-    depends_on:
-        - authelia-redis
-        - authelia-db
-    volumes:
-      - ./authelia/:/etc/authelia/:ro
-    deploy:
-      labels:
-        - "traefik.http.routers.authelia.rule=Host(`auth.example.org`)"
-        - "traefik.http.services.authelia.loadbalancer.server.port=9091"
-      restart_policy:
-        condition: on-failure
-        delay: 5s
-        max_attempts: 3
-        window: 120s
-
-  authelia-redis:
-    image: redis:5.0.7-alpine
-    command: ["redis-server", "--appendonly", "yes"]
-    volumes:
-      - ./authelia-redis/:/data/
-    networks:
-      - traefikbig
-    deploy:
-      restart_policy:
-        condition: on-failure
-        delay: 5s
-        max_attempts: 3
-        window: 120s
-
-  authelia-db:
-      image: postgres:12.2-alpine
-      environment:
-          POSTGRES_PASSWORD: z3A,hQ-9
-      volumes:
-          - ./docker_data/authelia_db/:/var/lib/postgresql/data/
-      networks:
-          - traefikbig
-      deploy:
-        restart_policy:
-          condition: on-failure
-          delay: 5s
-          max_attempts: 3
-          window: 120s
-
-networks:
-  traefikbig:
-    driver: overlay
-    name: traefikbig
-    ipam:
-      config:
-        - subnet: 10.50.0.0/16

templates/traefik_compose_template.yml.j2

-#jinja2: lstrip_blocks: “True”, trim_blocks: "True"
 version: '3.7'
 services:
   traefik:
@@ -89,7 +88,7 @@ services:
       - ./authelia/:/etc/authelia/:ro
     deploy:
       labels:
-        - "traefik.http.routers.authelia.rule=Host(`{{ auth_hostname }}`)"
+        - "traefik.http.routers.authelia.rule=Host(`{{ authentication_domain_name }}`)"
         - "traefik.http.services.authelia.loadbalancer.server.port=9091"
       restart_policy:
         condition: on-failure