diff --git a/README.md b/README.md
index 677c4433cb261b570656ede6705663e5db07472b..2d077bd2f1eac0e814e30ee1a337fab541f1a690 100755
--- a/README.md
+++ b/README.md
@@ -22,7 +22,7 @@ Traefik is a reverse proxy which allows to direct HTTP traffic to various Docker
The Traefik dashboard is deployed at [https://hostname/traefik/](https://hostname/traefik/)
Authelia is an authentication agent, which can be plugged to an LDAP server, and that Traefik can you to check permissions to access services.
-The authentication layer is optional. If used, the config file needs the variables `https_port`, `auth_hostname`, `authelia_config_path`.
+The authentication layer is optional. If used, the config file needs the variables `https_port`, `authentication_domain_name`, `authelia_config_path`.
Authelia is accessed automatically by Traefik to check permissions everytime someones wants to access a page.
If the user is not logged in, he is redirected to the authelia portal.
diff --git a/examples/authelia_config_example.yml b/examples/authelia_config.yml
similarity index 97%
rename from examples/authelia_config_example.yml
rename to examples/authelia_config.yml
index 1050c78eb53f58547efdb3b4d5a891c4df708242..8ce2b710d4844c77ed1e72fe14bab9b9db840726 100644
--- a/examples/authelia_config_example.yml
+++ b/examples/authelia_config.yml
@@ -16,7 +16,7 @@ log_level: info
# The secret used to generate JWT tokens when validating user identity by
# email confirmation.
# This secret can also be set using the env variables AUTHELIA_JWT_SECRET
-jwt_secret: XXXXXXXXXXXXXXXXX
+#jwt_secret: XXXXXXXXXXXXXXXXX
# Default redirection URL
#
@@ -82,7 +82,7 @@ authentication_backend:
# skip_verify: false
# The base dn for every entries
-# base_dn: dc=genouest,dc=org
+# base_dn: dc=domain,dc=org
# The attribute holding the username of the user. This attribute is used to populate
# the username in the session information. It was introduced due to #561 to handle case
@@ -196,7 +196,7 @@ access_control:
# Default policy can either be 'bypass', 'one_factor', 'two_factor' or 'deny'.
# It is the policy applied to any resource if there is no policy to be applied
# to the user.
- default_policy: bypass
+ default_policy: deny
rules:
# The login portal is freely accessible (redirectino loop otherwise)
@@ -213,12 +213,9 @@ access_control:
- domain: localhost
resources:
- "^/traefik/.*$"
+ - "^/api/.*$"
policy: one_factor
subject: "group:ldap_admin"
- - domain: localhost
- resources:
- - "^/traefik/.*$"
- policy: deny
# All galaxies are restricted to a group from ldap
- domain: localhost
@@ -237,22 +234,17 @@ access_control:
- "^/sp/genus_species/.*$"
policy: one_factor
subject: "group:gspecies"
- - domain: localhost
- resources:
- - "^/sp/genus_species/.*$"
- policy: deny
-
# Configuration of session cookies
#
# The session cookies identify the user once logged in.
session:
# The name of the session cookie. (default: authelia_session).
- name: authelia_replaceme_session
+ name: authelia_session
# The secret to encrypt the session data. This is only used with Redis.
# This secret can also be set using the env variables AUTHELIA_SESSION_SECRET
- secret: WXXXXXXXXXXXXXXXXXXXcXXXXXXXXXXXXXX
+# secret: WXXXXXXXXXXXXXXXXXXXcXXXXXXXXXXXXXX
# The time in seconds before the cookie expires and session is reset.
expiration: 3600000 # 1000 hour
@@ -271,7 +263,7 @@ session:
# The domain to protect.
# Note: the authenticator must also be in that domain. If empty, the cookie
# is restricted to the subdomain of the issuer.
- domain: replaceme.org
+ domain: domain.org
# The redis connection details
redis:
@@ -342,7 +334,7 @@ notifier:
host: smtp-server-hostname
port: 25
disable_require_tls: true
- sender: replace@me.fr
+ sender: replace@domain.org
# Sending an email using a Gmail account is as simple as the next section.
# You need to create an app password by following: https://support.google.com/accounts/answer/185833?hl=en
diff --git a/examples/authelia_secrets.env b/examples/authelia_secrets.env
new file mode 100644
index 0000000000000000000000000000000000000000..25485beada267503795b0d1ac875adf45069a6fd
--- /dev/null
+++ b/examples/authelia_secrets.env
@@ -0,0 +1,3 @@
+AUTHELIA_AUTHENTICATION_BACKEND_LDAP_PASSWORD=xxxxxxx
+AUTHELIA_JWT_SECRET=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+AUTHELIA_SESSION_SECRET=xxxxxxxxxxxxxxxxx
diff --git a/examples/example.yml b/examples/citrus_sinensis.yml
similarity index 100%
rename from examples/example.yml
rename to examples/citrus_sinensis.yml
diff --git a/examples/config_example.yml b/examples/config.yml
similarity index 68%
rename from examples/config_example.yml
rename to examples/config.yml
index 795cd662b0934475260f0dc88a6556399bd93929..b34b323103f734950a73c8de8ae4aa52bb73d497 100644
--- a/examples/config_example.yml
+++ b/examples/config.yml
@@ -3,12 +3,13 @@
# "all" section contains variables used by several services at once or the paths to import sensitive files
all:
hostname: localhost # Required. The hosting machine name
- dashboard_port: 8001 # Required. The desired port (on the hosting machine) for the traefik container dashboard
- http_port: 8888 # Required. The HTTP port docker will use on the hosting machine
+ http_port: 8888 # Required. The HTTP port docker will use on the hosting machine
https_port: 8889 # Required for Authelia. The HTTPS port docker will use on the hosting machine
- proxy_ip: XXXXXXXXXXXX # Required. IP of the upstream proxy (used by Traefik)
- authentication_domain_name: XXXXXXXXXXXX # Required for Authelia. The authentication domain name.
+ proxy_ip: XXX.XXX.XXX.XXX # Required. IP of the upstream proxy (used by Traefik)
+ authentication_domain_name: XXXXXXXX # Required for Authelia. The authentication domain name.
authelia_config_path: /path/to/authelia_config.yml # Required for Authelia. Path to the Authelia configuration file
+ authelia_secrets_env_path: /path/to/authelia/secrets.env # Required for Authelia. Path to the env file containing passwords and secrets needed for Authelia
+ authelia-db_postgres_password: XXXXXXXX # Required for Authelia.
# galaxy-specific variables
galaxy:
galaxy_default_admin_email: gga@galaxy.org # Required
@@ -17,11 +18,11 @@ galaxy:
webapollo_user: admin_apollo@galaxy.org # Required
webapollo_password: apollopass # Required
galaxy_config_remote_user_maildomain: mydomain.com # Required. The maildomain used by Galaxy authentication
- persist_galaxy_data: "True" # # Optional (default: True). If False, docker data will NOT be persisted on your host's file system and will be lost any time the galaxy container is recreated. Do not set this variable to "False" for production
+ persist_galaxy_data: "True" # Optional (default: True). If False, docker data will NOT be persisted on your host's file system and will be lost any time the galaxy container is recreated. Do not set this variable to "False" for production
# tripal-specific variables
tripal:
tripal_password: tripalpass # Required. Tripal database password (also used by galaxy as an environment variable)
- banner_path: /my/path/banner.png # Optional. Custom banner path
+ banner_path: /path/to/banner.png # Optional. Use this to change the top banner in Tripal
tripal_theme_name: tripal_gga # Optional. Use this to use another theme
tripal_theme_git_clone: http://gitlab.sb-roscoff.fr/abims/e-infra/tripal_gga.git # Optional. Use this to install another theme.
# jbrowse-specific variables
diff --git a/gga_init.py b/gga_init.py
index be5f3c3967accd1c5df1931145fcc4af4b14aaaf..fbefcebd1c2ce145db1270455cb8518ce3dd47a9 100755
--- a/gga_init.py
+++ b/gga_init.py
@@ -61,14 +61,11 @@ class DeploySpeciesStack(speciesData.SpeciesData):
# If the path specified is invalid (because it's empty or is still the default demo one),
# use the default banner instead
if "banner_path" in self.config.keys():
- if self.config["banner_path"] != "/path/to/banner" or self.config["banner_path"] != "":
- try:
- logging.debug("Custom banner path: %s" % self.config["banner_path"])
- if os.path.isfile(os.path.abspath(self.config["banner_path"])):
- shutil.copy(os.path.abspath(self.config["banner_path"]), "%s/banner.png" % self.species_dir)
- except FileNotFoundError:
- logging.warning("Specified banner not found (%s), using default banner instead" % self.config["banner_path"])
- self.config.pop("banner_path", None)
+ if not config["banner_path"] == "" and os.path.isfile(os.path.abspath(config["banner_path"])):
+ banner_dest_path = os.path.abspath("./banner.png")
+ logging.info("Custom banner path: %s" % self.config["banner_path"])
+ if os.path.samefile(os.path.abspath(config["banner_path"]),banner_dest_path):
+ shutil.copy(os.path.abspath(self.config["banner_path"]), banner_dest_path)
else:
logging.debug("Using default banner for Tripal pages")
self.config.pop("banner_path", None)
@@ -81,13 +78,13 @@ class DeploySpeciesStack(speciesData.SpeciesData):
try:
shutil.copy(os.path.join(self.script_dir, "files/nginx_download.conf"), os.path.abspath("./nginx/conf/default.conf"))
except Exception as exc:
- logging.critical("Could not copy nginx configuration file for %s" % self.full_name)
+ logging.critical("Could not copy nginx configuration file for %s %s", self.genus, self.species)
logging.critical(exc)
# Return to main directory
os.chdir(self.main_dir)
- logging.info("Directory tree generated for %s" % self.full_name)
+ logging.info("Directory tree generated for %s %s", self.genus, self.species)
def make_compose_files(self):
@@ -119,20 +116,20 @@ class DeploySpeciesStack(speciesData.SpeciesData):
render_vars = {**self.config, **input_vars}
# Render the gspecies docker-compose file and write it
- gspecies_compose_template = env.get_template("gspecies_compose_template.yml.j2")
+ gspecies_compose_template = env.get_template("gspecies_compose.yml.j2")
gspecies_compose_output = gspecies_compose_template.render(render_vars)
with open(os.path.join(self.species_dir, "docker-compose.yml"), "w") as gspecies_compose_file:
logging.info("Writing %s docker-compose.yml" % self.genus_species)
gspecies_compose_file.truncate(0)
gspecies_compose_file.write(gspecies_compose_output)
-
- galaxy_nginx_conf_template = env.get_template("galaxy_nginx.conf.j2")
- galaxy_nginx_conf_output = galaxy_nginx_conf_template.render(render_vars)
- with open(os.path.join(self.main_dir, "galaxy_nginx.conf"), "w") as galaxy_nginx_conf_file:
- logging.debug("Writing the galaxy_nginx.conf file for %s" % self.genus_species)
- galaxy_nginx_conf_file.truncate(0)
- galaxy_nginx_conf_file.write(galaxy_nginx_conf_output)
+ if not os.path.isfile(os.path.join(self.main_dir, "galaxy_nginx.conf")):
+ galaxy_nginx_conf_template = env.get_template("galaxy_nginx.conf.j2")
+ galaxy_nginx_conf_output = galaxy_nginx_conf_template.render(render_vars)
+ with open(os.path.join(self.main_dir, "galaxy_nginx.conf"), "w") as galaxy_nginx_conf_file:
+ logging.debug("Writing the galaxy_nginx.conf file for %s" % self.genus_species)
+ galaxy_nginx_conf_file.truncate(0)
+ galaxy_nginx_conf_file.write(galaxy_nginx_conf_output)
# Create the volumes (directory) of the species docker-compose file
create_mounts(working_dir=".", main_dir=self.main_dir)
@@ -199,10 +196,10 @@ def make_traefik_compose_files(config, main_dir):
# Jinja2 templating, handled using the python "jinja2" module
file_loader = FileSystemLoader(script_dir + "/templates")
- env = Environment(loader=file_loader)
+ env = Environment(loader=file_loader, trim_blocks=True, lstrip_blocks=True)
if not os.path.isfile("./traefik/docker-compose.yml"):
- traefik_compose_template = env.get_template("traefik_compose_template.yml.j2")
+ traefik_compose_template = env.get_template("traefik_compose.yml.j2")
traefik_compose_output = traefik_compose_template.render(render_vars)
with open(os.path.join(main_dir, "traefik/docker-compose.yml"), 'w') as traefik_compose_file:
logging.info("Writing traefik docker-compose.yml")
@@ -210,22 +207,12 @@ def make_traefik_compose_files(config, main_dir):
traefik_compose_file.write(traefik_compose_output)
if "authelia_config_path" in config.keys():
- if not config["authelia_config_path"] == "" or not config["authelia_config_path"] == "/path/to/authelia/config":
- if os.path.isfile(os.path.abspath(config["authelia_config_path"])):
- try:
- shutil.copy(os.path.abspath(config["authelia_config_path"]), "./traefik/authelia/configuration.yml")
- except Exception as exc:
- logging.critical("Could not copy authelia configuration file")
- sys.exit(exc)
- # authelia_config_template = env.get_template(os.path.basename(config["authelia_config_path"]))
- # authelia_config_output = authelia_config_template.render(render_vars)
- # with open(os.path.join(main_dir, "traefik/authelia/configuration.yml"), 'w') as authelia_config_file:
- # logging.info("Writing authelia configuration.yml")
- # authelia_config_file.truncate(0)
- # authelia_config_file.write(authelia_config_output)
- else:
- logging.critical("Cannot find authelia configuration path (%s)" % config["authelia_config_path"])
- sys.exit()
+ if not config["authelia_config_path"] == "" and os.path.isfile(os.path.abspath(config["authelia_config_path"])):
+ try:
+ shutil.copy(os.path.abspath(config["authelia_config_path"]), "./traefik/authelia/configuration.yml")
+ except Exception as exc:
+ logging.critical("Could not copy authelia configuration file")
+ sys.exit(exc)
else:
logging.critical("Invalid authelia configuration path (%s)" % config["authelia_config_path"])
sys.exit()
@@ -341,10 +328,7 @@ def deploy_stacks(input_list, main_dir, deploy_traefik):
if __name__ == "__main__":
- parser = argparse.ArgumentParser(description="Automatic data loading in containers and interaction "
- "with galaxy instances for GGA"
- ", following the protocol @ "
- "http://gitlab.sb-roscoff.fr/abims/e-infra/gga")
+ parser = argparse.ArgumentParser(description="Deploy GGA containers")
parser.add_argument("input",
type=str,
@@ -373,11 +357,8 @@ if __name__ == "__main__":
else:
logging.basicConfig(level=logging.INFO)
- # Parsing the config file if provided, using the default config otherwise
- if not args.config:
- args.config = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "config")
- else:
- args.config = os.path.abspath(args.config)
+ config_file = os.path.abspath(args.config)
+ config = utilities.parse_config(config_file)
main_dir = None
if not args.main_directory:
@@ -389,7 +370,6 @@ if __name__ == "__main__":
# Create traefik directory and compose files if needed or specified
if args.force_traefik or not os.path.isdir(os.path.join(os.path.abspath(main_dir), "traefik")):
- config = utilities.parse_config(args.config)
make_traefik_compose_files(config=config, main_dir=main_dir)
unique_sp_dict_list = utilities.get_unique_species_dict_list(sp_dict_list=sp_dict_list)
@@ -407,31 +387,19 @@ if __name__ == "__main__":
"/")
# Parse the config yaml file
- deploy_stack_for_current_organism.config = utilities.parse_config(args.config)
-
- # Set the instance url attribute
- for env_variable, value in deploy_stack_for_current_organism.config.items():
- if env_variable == "hostname":
- deploy_stack_for_current_organism.instance_url = value + \
- deploy_stack_for_current_organism.genus_lowercase + \
- "_" + deploy_stack_for_current_organism.species + \
- "/galaxy/"
- break
- else:
- deploy_stack_for_current_organism.instance_url = "http://localhost:8888/sp/{0}_{1}/galaxy/".format(
- deploy_stack_for_current_organism.genus_lowercase,
- deploy_stack_for_current_organism.species)
-
+ deploy_stack_for_current_organism.config = config
+
# Starting
- logging.info("gga_init.py called for %s" % deploy_stack_for_current_organism.full_name)
+ logging.info("gga_init.py called for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
+ logging.debug("Jbrowse url to %s %s %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species, deploy_stack_for_current_organism.sex, deploy_stack_for_current_organism.strain)
# Make/update directory tree
deploy_stack_for_current_organism.make_directory_tree()
- logging.info("Successfully generated the directory tree for %s" % deploy_stack_for_current_organism.full_name)
+ logging.info("Successfully generated the directory tree for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
# Make compose files
deploy_stack_for_current_organism.make_compose_files()
- logging.info("Successfully generated the docker-compose files for %s" % deploy_stack_for_current_organism.full_name)
+ logging.info("Successfully generated the docker-compose files for %s %s", deploy_stack_for_current_organism.genus, deploy_stack_for_current_organism.species)
logging.info("Deploying stacks")
if args.force_traefik:
diff --git a/serexec b/serexec
index 7a38c06c22d13eef5f39a0d5844875639af8d902..d73d97ef33ee0d6492d671b4ded2b7640bd065b8 100755
--- a/serexec
+++ b/serexec
@@ -4,11 +4,8 @@ set -e
SERVICE_NAME=$1; shift
TASK_ID=$(docker service ps --filter 'desired-state=running' $SERVICE_NAME -q)
-#we have only one node
-#NODE_ID=$(docker inspect --format '{{ .NodeID }}' $TASK_ID)
+#NODE_ID=$(docker inspect --format '{{ .NodeID }}' $TASK_ID) # if multiple nodes
CONTAINER_ID=$(docker inspect --format '{{ .Status.ContainerStatus.ContainerID }}' $TASK_ID)
-#we have only one node
-#NODE_HOST=$(docker node inspect --format '{{ .Description.Hostname }}' $NODE_ID)
-#we have only one node
-#export DOCKER_HOST="ssh://$USER@$NODE_HOST"
+#NODE_HOST=$(docker node inspect --format '{{ .Description.Hostname }}' $NODE_ID) # if multiple nodes
+#export DOCKER_HOST="ssh://$USER@$NODE_HOST" # if multiple nodes
docker exec -it $CONTAINER_ID "$@"
diff --git a/speciesData.py b/speciesData.py
index a2fcbdbfafabb742315fa73210b1c4f5fe102b7f..5d9ffc2a51abf5d4211232c9e6db09226729e286 100755
--- a/speciesData.py
+++ b/speciesData.py
@@ -15,14 +15,22 @@ class SpeciesData:
"""
+ def clean_string(self, string):
+ if not string is None and string != "":
+ clean_string = string.replace(" ", "_").replace("-", "_").replace("(", "").replace(")", "").replace("'", "").strip()
+ return clean_string
+ else:
+ return string
+
def __init__(self, parameters_dictionary):
# self.config_dictionary = None
self.parameters_dictionary = parameters_dictionary
- self.species = parameters_dictionary["description"]["species"].replace("(", "_").replace(")", "_").replace("-", "_")
- self.genus = parameters_dictionary["description"]["genus"].replace("(", "_").replace(")", "_").replace("-", "_")
- self.strain = parameters_dictionary["description"]["strain"].replace("(", "_").replace(")", "_").replace("-", "_")
- self.sex = parameters_dictionary["description"]["sex"].replace("(", "_").replace(")", "_").replace("-", "_")
- self.common = parameters_dictionary["description"]["common_name"].replace("(", "_").replace(")", "_").replace("-", "_")
+ parameters_dictionary_description=parameters_dictionary["description"]
+ self.species = self.clean_string(parameters_dictionary_description["species"])
+ self.genus = self.clean_string(parameters_dictionary_description["genus"])
+ self.strain = self.clean_string(parameters_dictionary_description["strain"])
+ self.sex = self.clean_string(parameters_dictionary_description["sex"])
+ self.common = self.clean_string(parameters_dictionary_description["common_name"])
self.date = datetime.today().strftime("%Y-%m-%d")
self.origin = parameters_dictionary["description"]["origin"]
diff --git a/templates/gspecies_compose_template.yml.j2 b/templates/gspecies_compose.yml.j2
similarity index 80%
rename from templates/gspecies_compose_template.yml.j2
rename to templates/gspecies_compose.yml.j2
index d897fa2d99d52a30ca9d2dd20db081612b80900f..e1fc127e79f5a6b20b58e426848a571022dcadc2 100644
--- a/templates/gspecies_compose_template.yml.j2
+++ b/templates/gspecies_compose.yml.j2
@@ -20,7 +20,7 @@ services:
- "traefik.http.routers.{{ genus_species }}-nginx.middlewares=sp-auth,sp-app-trailslash,sp-prefix"
{% else %}
- "traefik.http.routers.{{ genus_species }}-nginx.entryPoints=web"
- - "traefik.http.routers.{{ genus_species }}-nginx.middlewares=sp-app-trailslash,sp-prefix" # lg
+ - "traefik.http.routers.{{ genus_species }}-nginx.middlewares=sp-app-trailslash,sp-prefix"
{% endif %}
- "traefik.http.services.{{ genus_species }}-nginx.loadbalancer.server.port=80"
restart_policy:
@@ -38,7 +38,7 @@ services:
- ./docker_data/galaxy/:/export/:ro
- ./src_data/:/project_data/:ro
- ./src_data:/data:ro
- {% if 'banner' in render_vars %}
+ {% if 'banner_path' is defined %}
- ./banner.png:/var/www/html/banner.png:ro
{% endif %}
#- /groups/XXX/:/groups/XXX/:ro # We do this when we have symlinks in src_data pointing to /groups/XXX/...
@@ -53,24 +53,10 @@ services:
SITE_NAME: "{{ Genus_species }}"
ELASTICSEARCH_HOST: elasticsearch.{{ genus_species }}
ENABLE_JBROWSE: /jbrowse/?data=data/{{ genus_species_strain_sex }}
- # This ENABLE_JBROWSE variable should point to the "best assembly" by default --> tag it in the input file and use it to define this variable correctly (also called
- # unique id in the jbrowse tool parameters == both have to be identical)
- {% if apollo == True %}
- ENABLE_APOLLO: 1
- {% else %}
- ENABLE_APOLLO: 0
- {% endif %}
- {% if blast == True %}
- ENABLE_BLAST: 1
- {% else %}
- ENABLE_BLAST: 0
- {% endif %}
- ENABLE_DOWNLOAD: 1
- {% if wiki == True %}
- ENABLE_WIKI: 1
- {% else %}
- ENABLE_WIKI: 0
- {% endif %}
+ ENABLE_APOLLO: {{ apollo }}
+ ENABLE_BLAST: {{ blast }}
+ ENABLE_DOWNLOAD: {{ download }}
+ ENABLE_WIKI: {{ wiki }}
ENABLE_GO: 0
ENABLE_ORTHOLOGY: 0
ENABLE_ORTHOLOGY_LINKS: 0
@@ -89,7 +75,7 @@ services:
- "traefik.http.routers.{{ genus_species }}-tripal.middlewares=sp-auth,sp-trailslash,sp-prefix,tripal-addprefix"
{% else %}
- "traefik.http.routers.{{ genus_species }}-tripal.entryPoints=web"
- - "traefik.http.routers.{{ genus_species }}-tripal.middlewares=sp-trailslash,sp-prefix,tripal-addprefix" # lg
+ - "traefik.http.routers.{{ genus_species }}-tripal.middlewares=sp-trailslash,sp-prefix,tripal-addprefix"
{% endif %}
- "traefik.http.services.{{ genus_species }}-tripal.loadbalancer.server.port=80"
restart_policy:
@@ -112,14 +98,9 @@ services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:6.6.1
- #deploy:
- #resources:
- #limits:
- #memory: 500M
volumes:
- ./docker_data/elastic_search_index/:/usr/share/elasticsearch/data/
environment:
- # bootstrap.memory_lock: "true"
xpack.security.enabled: "false"
xpack.monitoring.enabled: "false"
xpack.ml.enabled: "false"
@@ -134,15 +115,11 @@ services:
galaxy:
image: quay.io/galaxy-genome-annotation/docker-galaxy-annotation:gmod
volumes:
- {% if persist_galaxy_data is defined %}
- {% if persist_galaxy_data == "False" %}
+ {% if (persist_galaxy_data is defined) and (persist_galaxy_data == "False") %}
#- ./docker_data/galaxy/:/export/
{% else %}
- ./docker_data/galaxy/:/export/
{% endif %}
- {% else %}
- - ./docker_data/galaxy/:/export/
- {% endif %}
- ./src_data/:/project_data/:ro
#- /groups/XXX/:/groups/XXX/:ro # We do this when we have symlinks in src_data pointing to /groups/XXX/...
- ./docker_data/jbrowse/:/jbrowse/data/
@@ -158,7 +135,7 @@ services:
GALAXY_DEFAULT_ADMIN_EMAIL: "{{ galaxy_default_admin_email }}"
GALAXY_DEFAULT_ADMIN_USER: "{{ galaxy_defaut_admin_user }}"
GALAXY_DEFAULT_ADMIN_PASSWORD: "{{ galaxy_default_admin_password }}"
- GALAXY_CONFIG_ADMIN_USERS: "admin@galaxy.org,{{ galaxy_default_admin_email }}" # admin@galaxy.org is the default (leave it), gogepp@bipaa is a shared ldap user we use to connect
+ GALAXY_CONFIG_ADMIN_USERS: "admin@galaxy.org,{{ galaxy_default_admin_email }}" # admin@galaxy.org is the default (leave it), galaxy_default_admin_email is a shared ldap user we use to connect
ENABLE_FIX_PERMS: 0
PROXY_PREFIX: /sp/{{ genus_species }}/galaxy
GALAXY_TRIPAL_URL: http://tripal.{{ genus_species }}/tripal/
@@ -186,12 +163,12 @@ services:
- "traefik.http.routers.{{ genus_species }}-galaxy.middlewares=sp-auth,sp-app-trailslash,sp-app-prefix"
{% else %}
- "traefik.http.routers.{{ genus_species }}-galaxy.entryPoints=web"
- - "traefik.http.routers.{{ genus_species }}-galaxy.middlewares=sp-app-trailslash,sp-app-prefix" #lg
+ - "traefik.http.routers.{{ genus_species }}-galaxy.middlewares=sp-app-trailslash,sp-app-prefix"
{% endif %}
- "traefik.http.services.{{ genus_species }}-galaxy.loadbalancer.server.port=80"
- "traefik.http.routers.{{ genus_species }}-gga_load-galaxy.rule=(Host(`localhost`) && PathPrefix(`/sp/{{ genus_species }}/galaxy`))"
- "traefik.http.routers.{{ genus_species }}-gga_load-galaxy.entryPoints=web"
-
+ - "traefik.http.routers.{{ genus_species }}-gga_load-galaxy.middlewares=sp-app-trailslash,sp-app-prefix"
restart_policy:
condition: on-failure
delay: 5s
@@ -217,7 +194,7 @@ services:
- "traefik.http.routers.{{ genus_species }}-jbrowse.middlewares=sp-auth,sp-app-trailslash,sp-app-prefix"
{% else %}
- "traefik.http.routers.{{ genus_species }}-jbrowse.entryPoints=web"
- - "traefik.http.routers.{{ genus_species }}-jbrowse.middlewares=sp-app-trailslash,sp-app-prefix" #lg
+ - "traefik.http.routers.{{ genus_species }}-jbrowse.middlewares=sp-app-trailslash,sp-app-prefix"
{% endif %}
- "traefik.http.services.{{ genus_species }}-jbrowse.loadbalancer.server.port=80"
restart_policy:
@@ -226,7 +203,7 @@ services:
max_attempts: 3
window: 120s
- {% if blast == True %}
+ {% if blast is defined and blast == 1 %}
blast:
image: quay.io/abretaud/sf-blast:latest
depends_on:
@@ -236,7 +213,7 @@ services:
UPLOAD_LIMIT: 20M
MEMORY_LIMIT: 128M
DB_NAME: 'postgres'
- ADMIN_EMAIL: 'g.ga@sb-roscoff.fr' # email sender
+ ADMIN_EMAIL: 'g.ga@domain.org' # email sender
ADMIN_NAME: 'gga' # email sender name
JOBS_METHOD: 'local' # Can be local (= no sge jobs, but run inside the container) or drmaa (= to submit to a cluster)
JOBS_WORK_DIR: '/tmp/blast_jobs/' # disk accessible both from compute nodes and mounted in this docker (at the same path)
@@ -252,8 +229,8 @@ services:
#JOBS_DRMAA_NATIVE: '-p web' # This line and following for slurm
#DRMAA_METHOD: 'slurm' # This line and following for slurm
volumes:
- - ../blast-themes/abims/:/var/www/blast/app/Resources/:ro # You can theme the app
- - /usr/local/genome2/:/usr/local/genome2/:ro # path for blast executables
+ - ../blast-themes/my_theme/:/var/www/blast/app/Resources/:ro # You can theme the app
+ - /path/to/blast/exe/:/path/to/blast/exe/:ro # path for blast executables
- /db/:/db/:ro # for access to indexed blast databases
#- /data1/sge/:/usr/local/sge/:ro # an sge install
#- /xxxx/blast_jobs/:/xxxx/blast_jobs/ # (for drmaa mode only)
@@ -277,7 +254,7 @@ services:
- "traefik.http.routers.{{ genus_species }}-blast.middlewares=sp-big-req,sp-auth,sp-app-trailslash,sp-app-prefix"
{% else %}
- "traefik.http.routers.{{ genus_species }}-blast.entryPoints=web"
- - "traefik.http.routers.{{ genus_species }}-blast.middlewares=sp-big-req,sp-app-trailslash,sp-app-prefix" # lg
+ - "traefik.http.routers.{{ genus_species }}-blast.middlewares=sp-big-req,sp-app-trailslash,sp-app-prefix"
{% endif %}
- "traefik.http.services.{{ genus_species }}-blast.loadbalancer.server.port=80"
restart_policy:
@@ -297,49 +274,9 @@ services:
- {{ genus_species }}
{% endif %}
- {% if wiki == True %}
- wiki:
- image: quay.io/abretaud/mediawiki
- environment:
- MEDIAWIKI_SERVER: http://localhost
- MEDIAWIKI_PROXY_PREFIX: /sp/{{ genus_species }}/wiki
- MEDIAWIKI_SITENAME: {{ Genus }} {{ species }}
- MEDIAWIKI_SECRET_KEY: XXXXXXXXXX
- MEDIAWIKI_DB_HOST: wiki-db.{{genus_species }}
- MEDIAWIKI_DB_PASSWORD: password
- MEDIAWIKI_ADMIN_USER: abretaud # ldap user
- depends_on:
- - wiki-db
- volumes:
- - ./docker_data/wiki_uploads:/images
- #- ../bipaa_wiki.png:/var/www/mediawiki/resources/assets/wiki.png:ro # To change the logo at the top left
- networks:
- - traefikbig
- - {{ genus_species }}
- deploy:
- labels:
- - "traefik.http.routers.{{ genus_species }}-wiki.rule=(Host(`{{ hostname }}`) && PathPrefix(`/sp/{{ genus_species }}/wiki`))"
- - "traefik.http.routers.{{ genus_species }}-wiki.tls=true"
- - "traefik.http.routers.{{ genus_species }}-wiki.entryPoints={{ entrypoint }}"
- - "traefik.http.routers.{{ genus_species }}-wiki.middlewares=sp-big-req,sp-auth,sp-app-trailslash,sp-app-prefix"
- - "traefik.http.services.{{ genus_species }}-wiki.loadbalancer.server.port=80"
- restart_policy:
- condition: on-failure
- delay: 5s
- max_attempts: 3
- window: 120s
-
- wiki-db:
- image: postgres:9.6-alpine
- volumes:
- - ./docker_data/wiki_db/:/var/lib/postgresql/data/
- networks:
- - {{ genus_species }}
- {% endif %}
-
networks:
traefikbig:
external: true
{{ genus_species }}:
driver: overlay
- name: {{ genus_species }}
+ name: {{ genus_species }}
\ No newline at end of file
diff --git a/templates/organisms.yml.j2 b/templates/organisms.yml.j2
new file mode 100644
index 0000000000000000000000000000000000000000..83ab58f9ab4bb3282af991df6ad0c50c952725f5
--- /dev/null
+++ b/templates/organisms.yml.j2
@@ -0,0 +1,25 @@
+{{ name }}:
+ description:
+ genus: {{ genus }}
+ species: {{ species }}
+ sex: {{ sex }}
+ strain: {{ strain }}
+ common_name: {{ common_name }}
+ origin: {{ origin }}
+ {% if main_species is defined and main_species is sameas true %}
+ main_species: yes
+ {% endif %}
+ data:
+ genome_path: {{ genome_path }}
+ transcripts_path: {{ transcripts_path }}
+ proteins_path: {{ proteins_path }}
+ gff_path: {{ gff_path }} # Mandatory!
+ interpro_path: {{ interpro_path }}
+ orthofinder_path: {{ orthofinder_path }}
+ blastp_path: {{ blastp_path }}
+ blastx_path: {{ blastx_path }}
+ genome_version: {{ genome_version }}
+ ogs_version: {{ ogs_version }}
+ performed_by: {{ performed_by }}
+ services:
+ blast: {{ blast }}
\ No newline at end of file
diff --git a/templates/orthology_compose_template.yml.j2 b/templates/orthology_compose.yml.j2
similarity index 100%
rename from templates/orthology_compose_template.yml.j2
rename to templates/orthology_compose.yml.j2
diff --git a/templates/traefik_compose_template.yml.j2 b/templates/traefik_compose.yml.j2
similarity index 81%
rename from templates/traefik_compose_template.yml.j2
rename to templates/traefik_compose.yml.j2
index 6157cc2acb31627e05c6b9105370980ea5b6ab3d..a707d1a2dc6c93626a93273d499ecab8adc7e775 100644
--- a/templates/traefik_compose_template.yml.j2
+++ b/templates/traefik_compose.yml.j2
@@ -5,19 +5,15 @@ services:
command:
- "--api"
- "--api.dashboard"
-# - "--api.insecure=true" # added by lg to debug, for dashboard
- "--log.level=DEBUG"
- "--providers.docker"
- "--providers.docker.swarmMode=true"
- - "--providers.docker.network=traefikbig" # changed by lg from traefik to traefikbig
+ - "--providers.docker.network=traefikbig"
- "--entryPoints.web.address=:80"
- - "--entryPoints.web.forwardedHeaders.trustedIPs={{ proxy_ip }}" # The ips of our upstream proxies: eci
+ - "--entryPoints.web.forwardedHeaders.trustedIPs={{ proxy_ip }}" # The ips of our upstream proxies
- "--entryPoints.webs.address=:443"
- - "--entryPoints.webs.forwardedHeaders.trustedIPs={{ proxy_ip }}" # The ips of our upstream proxies: eci
+ - "--entryPoints.webs.forwardedHeaders.trustedIPs={{ proxy_ip }}" # The ips of our upstream proxies
ports:
- {% if dashboard_port is defined %}
- - {{ dashboard_port }}:8080 # added by lg to debug, for dashboard
- {% endif %}
- {{ http_port }}:80
{% if https_port is defined %}
- {{ https_port }}:443
@@ -31,37 +27,39 @@ services:
constraints:
- node.role == manager
labels:
-# - "traefik.http.routers.traefik-api.rule=PathPrefix(`/traefik`)"
- - "traefik.http.routers.traefik-api.rule=PathPrefix(`/api`) || PathPrefix(`/dashboard`) || PathPrefix(`/traefik`)" # lg
+ - "traefik.http.routers.traefik-api.rule=PathPrefix(`/api`) || PathPrefix(`/dashboard`) || PathPrefix(`/traefik`)"
{% if https_port is defined %}
- "traefik.http.routers.traefik-api.tls=true"
- "traefik.http.routers.traefik-api.entryPoints=webs"
{% else %}
- - "traefik.http.routers.traefik-api.entryPoints=web" # lg
+ - "traefik.http.routers.traefik-api.entryPoints=web"
{% endif %}
- "traefik.http.routers.traefik-api.service=api@internal"
- "traefik.http.middlewares.traefik-strip.stripprefix.prefixes=/traefik"
- - "traefik.http.middlewares.traefik-auth.forwardauth.address=http://authelia:9091/api/verify?rd=https://auth.abims-gga.sb-roscoff.fr/"
+ - "traefik.http.middlewares.traefik-auth.forwardauth.address=http://authelia:9091/api/verify?rd=https://{{ authentication_domain_name }}/"
- "traefik.http.middlewares.traefik-auth.forwardauth.trustForwardHeader=true"
-# - "traefik.http.routers.traefik-api.middlewares=traefik-auth,traefik-strip"
- - "traefik.http.routers.traefik-api.middlewares=traefik-strip" # lg
+ {% if https_port is defined %}
+ - "traefik.http.routers.traefik-api.middlewares=traefik-auth,traefik-strip"
+ {% else %}
+ - "traefik.http.routers.traefik-api.middlewares=traefik-strip"
+ {% endif %}
# Dummy service for Swarm port detection. The port can be any valid integer value.
- "traefik.http.services.traefik-svc.loadbalancer.server.port=9999"
# Some generally useful middlewares for organisms hosting
- - "traefik.http.middlewares.sp-auth.forwardauth.address=http://authelia:9091/api/verify?rd=https://auth.abims-gga.sb-roscoff.fr/"
+ - "traefik.http.middlewares.sp-auth.forwardauth.address=http://authelia:9091/api/verify?rd=https://{{ authentication_domain_name }}/"
- "traefik.http.middlewares.sp-auth.forwardauth.trustForwardHeader=true"
- "traefik.http.middlewares.sp-auth.forwardauth.authResponseHeaders=Remote-User,Remote-Groups"
{% if https_port is defined %}
- "traefik.http.middlewares.sp-trailslash.redirectregex.regex=^(https?://[^/]+/sp/[^/]+)$$"
{% else %}
- - "traefik.http.middlewares.sp-trailslash.redirectregex.regex=^(http?://[^/]+/sp/[^/]+)$$" # lg
+ - "traefik.http.middlewares.sp-trailslash.redirectregex.regex=^(http?://[^/]+/sp/[^/]+)$$"
{% endif %}
- "traefik.http.middlewares.sp-trailslash.redirectregex.replacement=$${1}/"
- "traefik.http.middlewares.sp-trailslash.redirectregex.permanent=true"
{% if https_port is defined %}
- "traefik.http.middlewares.sp-app-trailslash.redirectregex.regex=^(https?://[^/]+/sp/[^/]+/[^/]+)$$"
{% else %}
- - "traefik.http.middlewares.sp-app-trailslash.redirectregex.regex=^(http?://[^/]+/sp/[^/]+/[^/]+)$$" # lg
+ - "traefik.http.middlewares.sp-app-trailslash.redirectregex.regex=^(http?://[^/]+/sp/[^/]+/[^/]+)$$"
{% endif %}
- "traefik.http.middlewares.sp-app-trailslash.redirectregex.replacement=$${1}/"
- "traefik.http.middlewares.sp-app-trailslash.redirectregex.permanent=true"
@@ -86,6 +84,10 @@ services:
- authelia-db
volumes:
- ./authelia/:/etc/authelia/:ro
+ {% if authelia_secrets_env_path is defined %}
+ env_file:
+ - ./authelia/secrets.env
+ {% endif %}
deploy:
labels:
- "traefik.http.routers.authelia.rule=Host(`{{ authentication_domain_name }}`)"
@@ -113,7 +115,7 @@ services:
authelia-db:
image: postgres:12.2-alpine
environment:
- POSTGRES_PASSWORD: z3A,hQ-9
+ POSTGRES_PASSWORD: {{ authelia-db_postgres_password }}
volumes:
- ./docker_data/authelia_db/:/var/lib/postgresql/data/
networks:
@@ -132,4 +134,4 @@ networks:
name: traefikbig
ipam:
config:
- - subnet: 10.50.0.0/16
+ - subnet: 10.50.0.0/16
\ No newline at end of file
diff --git a/utilities.py b/utilities.py
index d6b8504c65a98749c311de30847f793a6c574d43..2bb532d727c8f6cb704695d5ba374ff18a7be12f 100755
--- a/utilities.py
+++ b/utilities.py
@@ -123,6 +123,16 @@ def get_species_history_id(instance, full_name):
return [history_id, show_history]
+def get_gspecies_string_from_sp_dict(sp_dict):
+
+ gspecies = ""
+ for k, v in sp_dict.items():
+ for k2, v2 in v.items():
+ if k2 == "genus":
+ gspecies = gspecies.lower() + v2
+ elif k2 == "species":
+ gspecies = gspecies.lower() + "_" + v2
+ return gspecies
def get_unique_species_str_list(sp_dict_list):
"""
@@ -137,15 +147,9 @@ def get_unique_species_str_list(sp_dict_list):
unique_species_li = []
for sp in sp_dict_list:
- for k, v in sp.items():
- sp_gspecies = ""
- for k2, v2 in v.items():
- if k2 == "genus":
- sp_gspecies = sp_gspecies.lower() + v2
- elif k2 == "species":
- sp_gspecies = sp_gspecies.lower() + "_" + v2
- if sp_gspecies not in unique_species_li and sp_gspecies != "":
- unique_species_li.append(sp_gspecies)
+ sp_gspecies = get_gspecies_string_from_sp_dict(sp)
+ if sp_gspecies not in unique_species_li and sp_gspecies != "":
+ unique_species_li.append(sp_gspecies)
return unique_species_li
@@ -165,11 +169,11 @@ def get_unique_species_dict_list(sp_dict_list):
unique_species_genus_species = get_unique_species_str_list(sp_dict_list=sp_dict_list)
for sp in sp_dict_list:
- for gspecies in unique_species_genus_species:
- if gspecies not in unique_species_dict.keys():
- unique_species_dict[gspecies] = sp
- else:
- continue
+ gspecies = get_gspecies_string_from_sp_dict(sp)
+ if gspecies not in unique_species_dict.keys() or sp["description"]["main_species"] == True :
+ unique_species_dict[gspecies] = sp
+ else:
+ continue
for k, v in unique_species_dict.items():
unique_species_list_of_dict.append(v)
diff --git a/workflows/Blast_Diamond.ga b/workflows_phaeoexplorer/Blast_Diamond.ga
similarity index 100%
rename from workflows/Blast_Diamond.ga
rename to workflows_phaeoexplorer/Blast_Diamond.ga
diff --git a/workflows/Chado_load_Tripal_synchronize.ga b/workflows_phaeoexplorer/Chado_load_Tripal_synchronize.ga
similarity index 100%
rename from workflows/Chado_load_Tripal_synchronize.ga
rename to workflows_phaeoexplorer/Chado_load_Tripal_synchronize.ga
diff --git a/workflows/Chado_load_Tripal_synchronize.ga.bak b/workflows_phaeoexplorer/Chado_load_Tripal_synchronize.ga.bak
similarity index 100%
rename from workflows/Chado_load_Tripal_synchronize.ga.bak
rename to workflows_phaeoexplorer/Chado_load_Tripal_synchronize.ga.bak
diff --git a/workflows/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_1org_v1.ga b/workflows_phaeoexplorer/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_1org_v1.ga
similarity index 100%
rename from workflows/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_1org_v1.ga
rename to workflows_phaeoexplorer/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_1org_v1.ga
diff --git a/workflows/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_1org_v2.ga b/workflows_phaeoexplorer/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_1org_v2.ga
similarity index 100%
rename from workflows/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_1org_v2.ga
rename to workflows_phaeoexplorer/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_1org_v2.ga
diff --git a/workflows/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_2org_v1.ga b/workflows_phaeoexplorer/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_2org_v1.ga
similarity index 100%
rename from workflows/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_2org_v1.ga
rename to workflows_phaeoexplorer/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_2org_v1.ga
diff --git a/workflows/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_2org_v2.ga b/workflows_phaeoexplorer/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_2org_v2.ga
similarity index 100%
rename from workflows/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_2org_v2.ga
rename to workflows_phaeoexplorer/Galaxy-Workflow-chado_load_tripal_synchronize_jbrowse_2org_v2.ga
diff --git a/workflows/Interproscan.ga b/workflows_phaeoexplorer/Interproscan.ga
similarity index 100%
rename from workflows/Interproscan.ga
rename to workflows_phaeoexplorer/Interproscan.ga
diff --git a/workflows/Jbrowse.ga b/workflows_phaeoexplorer/Jbrowse.ga
similarity index 100%
rename from workflows/Jbrowse.ga
rename to workflows_phaeoexplorer/Jbrowse.ga