From e0b28ab6a2331e2b94d97955783dbf35557c62c0 Mon Sep 17 00:00:00 2001
From: Arthur Le Bars <arthur.le-bars@sb-roscoff.fr>
Date: Thu, 8 Apr 2021 13:49:14 +0200
Subject: [PATCH] clearer authentication domain name variable, changed jbrowse
menu_url assign
---
examples/config_example.yml | 8 +-
run_workflow_phaeoexplorer.py | 7 +-
templates/gspecies_compose_template.yml.bak | 303 ++++++++++++++++++++
templates/traefik_compose_template.yml.j2 | 2 +-
4 files changed, 313 insertions(+), 7 deletions(-)
create mode 100644 templates/gspecies_compose_template.yml.bak
diff --git a/examples/config_example.yml b/examples/config_example.yml
index 21d0ac6..6f22504 100644
--- a/examples/config_example.yml
+++ b/examples/config_example.yml
@@ -5,10 +5,10 @@ all:
hostname: localhost # The hosting machine name
dashboard_port: 8001 # The desired port (on the hosting machine) for the traefik container dashboard
http_port: 8888 # The HTTP port docker will use on the hosting machine
- https_port: 8889 # The HTTPS port docker will use on the hosting machine. Required for Authelia
+ https_port: 8889 # Optional. The HTTPS port docker will use on the hosting machine. Required for Authelia
proxy_ip: XXXXXXXXXXXX # IP of the upstream proxy (used by Traefik)
- auth_hostname: XXXXXXXXXXXX # Required for Authelia. The authentication domain name.
- authelia_config_path: /path/to/authelia_config.yml # Required for Authelia. Path to the Authelia configuration file
+ authentication_domain_name: XXXXXXXXXXXX # Optional. Required for Authelia. The authentication domain name.
+ authelia_config_path: /path/to/authelia_config.yml # Optional. Required for Authelia. Path to the Authelia configuration file
# galaxy-specific variables
galaxy:
galaxy_default_admin_email: gga@galaxy.org
@@ -26,4 +26,4 @@ tripal:
tripal_theme_git_clone: http://gitlab.sb-roscoff.fr/abims/e-infra/tripal_gga.git # Optional. Use this to install another theme.
# jbrowse-specific variables
jbrowse:
- menu_url: "http://localhost:8888/" # Used with run_workflow_phaeoexplorer.py. If present, this variable is used to define jbrowse menu_url, if absent, will use https://hostname:https_port instead
\ No newline at end of file
+ menu_url: "http://localhost:8888/" # Optional. Used with run_workflow_phaeoexplorer.py. If present, this variable is used to define jbrowse menu_url, if absent, will use https://hostname:https_port instead
\ No newline at end of file
diff --git a/run_workflow_phaeoexplorer.py b/run_workflow_phaeoexplorer.py
index 3907a43..e97f1fe 100755
--- a/run_workflow_phaeoexplorer.py
+++ b/run_workflow_phaeoexplorer.py
@@ -492,9 +492,12 @@ class RunWorkflow(speciesData.SpeciesData):
# In case of the Jbrowse workflow, we unfortunately have to manually edit the parameters instead of setting them
# as runtime values, using runtime parameters makes the tool throw an internal critical error ("replace not found" error)
# Scratchgmod test: need "http" (or "https"), the hostname (+ port)
- # TODO --> use menu_url variable in config to set up the menu url
+ if not self.config["menu_url"]:
+ jbrowse_menu_url = "https://{hostname}/sp/{genus_sp}/feature/{Genus}/{species}/{id}".format(hostname=self.config["hostname"], genus_sp=self.genus_species, Genus=self.genus_uppercase, species=self.species, id="{id}")
+ else:
+ jbrowse_menu_url = self.config["menu_url"]
if workflow_name == "Jbrowse":
- workflow_dict["steps"]["2"]["tool_state"] = workflow_dict["steps"]["2"]["tool_state"].replace("__MENU_URL__", "https://{hostname}/sp/{genus_sp}/feature/{Genus}/{species}/{id}".format(hostname=self.config["hostname"], genus_sp=self.genus_species, Genus=self.genus_uppercase, species=self.species, id="{id}"))
+ workflow_dict["steps"]["2"]["tool_state"] = workflow_dict["steps"]["2"]["tool_state"].replace("__MENU_URL__", jbrowse_menu_url)
# The UNIQUE_ID is specific to a combination genus_species_strain_sex so every combination should have its unique workflow
# in galaxy --> define a naming method for these workflows
workflow_dict["steps"]["3"]["tool_state"] = workflow_dict["steps"]["3"]["tool_state"].replace("__FULL_NAME__", self.full_name).replace("__UNIQUE_ID__", self.species_folder_name)
diff --git a/templates/gspecies_compose_template.yml.bak b/templates/gspecies_compose_template.yml.bak
new file mode 100644
index 0000000..da6d645
--- /dev/null
+++ b/templates/gspecies_compose_template.yml.bak
@@ -0,0 +1,303 @@
+# ./docker_data is created and filled with persistent data that should be backuped
+
+version: '3.7'
+services:
+ proxy:
+ image: quay.io/abretaud/nginx-ldap:latest
+ volumes:
+ - ./src_data/:/project_data/
+ #- /groups/XXX/:/groups/XXX/:ro # We do this when we have symlinks in src_data pointing to /groups/XXX/...
+ - ./nginx/conf:/etc/nginx/conf.d
+ networks:
+ - traefikbig
+ - genus_species
+ deploy:
+ labels:
+ # Download page
+ - "traefik.http.routers.genus_species-nginx.rule=(Host(`custom_host`) && PathPrefix(`/sp/genus_species/download`))"
+# - "traefik.http.routers.genus_species-nginx.tls=true"
+# - "traefik.http.routers.genus_species-nginx.entryPoints=webs"
+ - "traefik.http.routers.genus_species-nginx.entryPoints=web" # lg
+# - "traefik.http.routers.genus_species-nginx.middlewares=sp-auth,sp-app-trailslash,sp-prefix"
+ - "traefik.http.routers.genus_species-nginx.middlewares=sp-app-trailslash,sp-prefix" # lg
+ - "traefik.http.services.genus_species-nginx.loadbalancer.server.port=80"
+ restart_policy:
+ condition: on-failure
+ delay: 5s
+ max_attempts: 3
+ window: 120s
+
+ tripal:
+ image: quay.io/galaxy-genome-annotation/tripal:v2.x
+ depends_on:
+ - tripal-db
+ - elasticsearch
+ volumes:
+ - ./docker_data/galaxy/:/export/:ro
+ - ./src_data/:/project_data/:ro
+ - ./src_data:/data:ro
+ - ./banner.png:/var/www/html/banner.png:ro
+ #- /groups/XXX/:/groups/XXX/:ro # We do this when we have symlinks in src_data pointing to /groups/XXX/...
+ environment:
+ DB_HOST: tripal-db.genus_species
+ BASE_URL_PATH: /sp/genus_species
+ UPLOAD_LIMIT: 20M
+ MEMORY_LIMIT: 512M
+ TRIPAL_GIT_CLONE_MODULES: "https://github.com/abretaud/tripal_rest_api.git[@c6f9021ea5d4c6d7c67c5bd363a7dd9359228bbc] https://github.com/tripal/tripal_elasticsearch.git[@dc7f276046e394a80a7dfc9404cf1a149006eb2a] https://github.com/tripal/tripal_analysis_interpro.git https://github.com/tripal/tripal_analysis_go.git https://github.com/tripal/tripal_analysis_blast.git https://github.com/tripal/tripal_analysis_expression.git[@7240039fdeb4579afd06bbcb989cb7795bd4c342]"
+ TRIPAL_DOWNLOAD_MODULES: ""
+ TRIPAL_ENABLE_MODULES: "tripal_analysis_blast tripal_analysis_interpro tripal_analysis_go tripal_rest_api tripal_elasticsearch"
+ SITE_NAME: "Genus species"
+ ELASTICSEARCH_HOST: elasticsearch.genus_species
+ ENABLE_JBROWSE: /jbrowse/?data=data/genus_species_sex
+ ENABLE_APOLLO: 0
+ ENABLE_BLAST: 1
+ ENABLE_DOWNLOAD: 1
+ ENABLE_WIKI: 0
+ ENABLE_GO: 0
+ ENABLE_ORTHOLOGY: 0
+ ENABLE_ORTHOLOGY_LINKS: 0
+ THEME: "custom_theme" # Use this to use another theme
+ THEME_GIT_CLONE: "custom_git_clone" # Use this to install another theme
+ ADMIN_PASSWORD: custom_tripal_admin_password # You need to define it and update it in galaxy config below
+ networks:
+ - traefikbig
+ - genus_species
+ deploy:
+ labels:
+ - "traefik.http.routers.genus_species-tripal.rule=(Host(`custom_host`) && PathPrefix(`/sp/genus_species`))"
+# - "traefik.http.routers.genus_species-tripal.tls=true"
+# - "traefik.http.routers.genus_species-tripal.entryPoints=webs"
+ - "traefik.http.routers.genus_species-tripal.entryPoints=web" # lg
+# - "traefik.http.routers.genus_species-tripal.middlewares=sp-auth,sp-trailslash,sp-prefix,tripal-addprefix"
+ - "traefik.http.routers.genus_species-tripal.middlewares=sp-trailslash,sp-prefix,tripal-addprefix" # lg
+ - "traefik.http.services.genus_species-tripal.loadbalancer.server.port=80"
+ restart_policy:
+ condition: on-failure
+ delay: 5s
+ max_attempts: 3
+ window: 120s
+
+ tripal-db:
+ image: quay.io/galaxy-genome-annotation/chado:1.31-jenkins26-pg9.5
+ environment:
+ - POSTGRES_PASSWORD=postgres
+ # The default chado image would try to install the schema on first run,
+ # we just want the tools to be available.
+ - INSTALL_CHADO_SCHEMA=0
+ volumes:
+ - ./docker_data/tripal_db/:/var/lib/postgresql/data/
+ networks:
+ - genus_species
+
+ elasticsearch:
+ image: docker.elastic.co/elasticsearch/elasticsearch:6.6.1
+ #deploy:
+ #resources:
+ #limits:
+ #memory: 500M
+ volumes:
+ - ./docker_data/elastic_search_index/:/usr/share/elasticsearch/data/
+ environment:
+ bootstrap.memory_lock: "true"
+ xpack.security.enabled: "false"
+ xpack.monitoring.enabled: "false"
+ xpack.ml.enabled: "false"
+ xpack.graph.enabled: "false"
+ xpack.watcher.enabled: "false"
+ cluster.routing.allocation.disk.threshold_enabled: "false"
+ ES_JAVA_OPTS: "-Xms500m -Xmx500m"
+ TAKE_FILE_OWNERSHIP: "true"
+ networks:
+ - genus_species
+
+ galaxy:
+ image: quay.io/galaxy-genome-annotation/docker-galaxy-annotation:gmod
+ volumes:
+ - ../galaxy_data_libs_SI.py:/opt/galaxy_data_libs_SI.py
+ #- ./docker_data/galaxy/:/export/ # Commented for testing TODO: Uncomment in production
+ - ./src_data/:/project_data/:ro
+ #- /groups/XXX/:/groups/XXX/:ro # We do this when we have symlinks in src_data pointing to /groups/XXX/...
+ - ./docker_data/jbrowse/:/jbrowse/data/
+ - ./docker_data/apollo/:/apollo-data/
+ - ../galaxy_nginx.conf:/etc/nginx/uwsgi_params
+ environment:
+ NONUSE: nodejs,proftp,reports,condor
+ GALAXY_LOGGING: full
+ GALAXY_CONFIG_BRAND: "Genus species"
+ GALAXY_CONFIG_ALLOW_LIBRARY_PATH_PASTE: "True"
+ GALAXY_CONFIG_USE_REMOTE_USER: "True"
+ GALAXY_CONFIG_REMOTE_USER_MAILDOMAIN: "sb-roscoff.fr"
+ GALAXY_DEFAULT_ADMIN_EMAIL: "custom_galaxy_default_admin_email"
+ GALAXY_DEFAULT_ADMIN_USER: "custom_galaxy_defaut_admin_user"
+ GALAXY_DEFAULT_ADMIN_PASSWORD: "custom_galaxy_default_admin_password"
+ GALAXY_CONFIG_ADMIN_USERS: "admin@galaxy.org, gga@sb-roscoff.fr, lgueguen@sb-roscoff.fr, alebars@sb-roscoff.fr" # admin@galaxy.org is the default (leave it), gogepp@bipaa is a shared ldap user we use to connect
+ GALAXY_CONFIG_MASTER_API_KEY: "custom_galaxy_config_master_api_key"
+ ENABLE_FIX_PERMS: 0
+ PROXY_PREFIX: /sp/genus_species/galaxy
+ GALAXY_TRIPAL_URL: http://tripal.genus_species/tripal/
+ GALAXY_TRIPAL_PASSWORD: custom_galaxy_tripal_password # See tripal config above
+ GALAXY_WEBAPOLLO_URL: http://one-of-the-swarm-node:8888/apollo/
+ GALAXY_WEBAPOLLO_USER: "custom_galaxy_web_apollo_user"
+ GALAXY_WEBAPOLLO_PASSWORD: "custom_galaxy_web_apollo_password" # See tripal config below
+ GALAXY_WEBAPOLLO_EXT_URL: /apollo/
+ GALAXY_CHADO_DBHOST: tripal-db.genus_species
+ GALAXY_CHADO_DBSCHEMA: chado
+ GALAXY_AUTO_UPDATE_DB: 1
+ GALAXY_AUTO_UPDATE_CONDA: 1
+ GALAXY_AUTO_UPDATE_TOOLS: "/galaxy-central/tools_1.yaml"
+ GALAXY_SHARED_DIR: ""
+ BLAT_ENABLED: 1
+ networks:
+ - traefikbig
+ - genus_species
+ deploy:
+ labels:
+ - "traefik.http.routers.genus_species-galaxy.rule=(Host(`custom_host`) && PathPrefix(`/sp/genus_species/galaxy`))"
+# - "traefik.http.routers.genus_species-galaxy.tls=true"
+# - "traefik.http.routers.genus_species-galaxy.entryPoints=webs"
+ - "traefik.http.routers.genus_species-galaxy.entryPoints=web" #lg
+# - "traefik.http.routers.genus_species-galaxy.middlewares=sp-auth,sp-app-trailslash,sp-app-prefix"
+ - "traefik.http.routers.genus_species-galaxy.middlewares=sp-app-trailslash,sp-app-prefix" #lg
+ - "traefik.http.services.genus_species-galaxy.loadbalancer.server.port=80"
+ restart_policy:
+ condition: on-failure
+ delay: 5s
+ max_attempts: 3
+ window: 120s
+
+ jbrowse:
+ image: quay.io/galaxy-genome-annotation/jbrowse:v1.16.8
+ volumes:
+ - ./docker_data/galaxy/:/export/:ro
+ - ./src_data/:/project_data/:ro
+ #- /groups/XXX/:/groups/XXX/:ro # We do this when we have symlinks in src_data pointing to /groups/XXX/...
+ - ./docker_data/jbrowse/:/jbrowse/data/:ro
+ networks:
+ - traefikbig
+ - genus_species
+ deploy:
+ labels:
+ - "traefik.http.routers.genus_species-jbrowse.rule=(Host(`custom_host`) && PathPrefix(`/sp/genus_species/jbrowse`))"
+# - "traefik.http.routers.genus_species-jbrowse.tls=true"
+# - "traefik.http.routers.genus_species-jbrowse.entryPoints=webs"
+ - "traefik.http.routers.genus_species-jbrowse.entryPoints=web" # lg
+# - "traefik.http.routers.genus_species-jbrowse.middlewares=sp-auth,sp-app-trailslash,sp-app-prefix"
+ - "traefik.http.routers.genus_species-jbrowse.middlewares=sp-app-trailslash,sp-app-prefix" #lg
+ - "traefik.http.services.genus_species-jbrowse.loadbalancer.server.port=80"
+ restart_policy:
+ condition: on-failure
+ delay: 5s
+ max_attempts: 3
+ window: 120s
+
+ blast:
+ image: quay.io/abretaud/sf-blast:latest
+ depends_on:
+ - blast-db
+ environment:
+ DB_HOST: blast-db.genus_species
+ UPLOAD_LIMIT: 20M
+ MEMORY_LIMIT: 128M
+ DB_NAME: 'postgres'
+ ADMIN_EMAIL: 'g.ga@sb-roscoff.fr' # email sender
+ ADMIN_NAME: 'gga' # email sender name
+ JOBS_METHOD: 'local' # Can be local (= no sge jobs, but run inside the container) or drmaa (= to submit to a cluster)
+ JOBS_WORK_DIR: '/tmp/blast_jobs/' # disk accessible both from compute nodes and mounted in this docker (at the same path)
+ CDD_DELTA_PATH: '/db/cdd_delta/current/flat/cdd_delta'
+ BLAST_TITLE: 'Genus species blast server'
+ JOBS_SCHED_NAME: 'blast_gspecies' # job names
+ #PRE_CMD: '. /local/env/envblast-2.6.0.sh; . /local/env/envpython-3.7.1.sh;' # executed at the beginning of each job
+ #APACHE_RUN_USER: 'bipaaweb' # username known by sge
+ #APACHE_RUN_GROUP: 'bipaa' # group known by sge
+ BASE_URL_PATH: '/sp/genus_species/blast/'
+ UID: 55914 # username known by sge (for drmaa mode only)
+ GID: 40259 # group known by sge (for drmaa mode only)
+ #JOBS_DRMAA_NATIVE: '-p web' # This line and following for slurm
+ #DRMAA_METHOD: 'slurm' # This line and following for slurm
+ volumes:
+ - ../blast-themes/abims/:/var/www/blast/app/Resources/:ro # You can theme the app
+ - /usr/local/genome2/:/usr/local/genome2/:ro # path for blast executables
+ - /db/:/db/:ro # for access to indexed blast databases
+ #- /data1/sge/:/usr/local/sge/:ro # an sge install
+ #- /xxxx/blast_jobs/:/xxxx/blast_jobs/ # (for drmaa mode only)
+ - ./blast/banks.yml:/var/www/blast/app/config/banks.yml:ro
+ - ./blast/links.yml:/etc/blast_links/links.yml:ro
+ #- /data1/slurm/slurm.conf:/etc/slurm-llnl/slurm.conf:ro # This line and following for slurm
+ #- /data1/slurm/gres.conf:/etc/slurm-llnl/gres.conf:ro
+ #- /data1/slurm/cgroup.conf:/etc/slurm-llnl/cgroup.conf:ro
+ #- /data1/slurm/slurmdbd.conf:/etc/slurm-llnl/slurmdbd.conf:ro
+ #- /data1/slurm/drmaa/:/etc/slurm-llnl/drmaa/:ro
+ #- /etc/munge/:/etc/munge/:ro
+ networks:
+ - traefikbig
+ - genus_species
+ deploy:
+ labels:
+ - "traefik.http.routers.genus_species-blast.rule=(Host(`custom_host`) && PathPrefix(`/sp/genus_species/blast`))"
+# - "traefik.http.routers.genus_species-blast.tls=true"
+# - "traefik.http.routers.genus_species-blast.entryPoints=webs"
+ - "traefik.http.routers.genus_species-blast.entryPoints=web" # lg
+# - "traefik.http.routers.genus_species-blast.middlewares=sp-big-req,sp-auth,sp-app-trailslash,sp-app-prefix"
+ - "traefik.http.routers.genus_species-blast.middlewares=sp-big-req,sp-app-trailslash,sp-app-prefix" # lg
+ - "traefik.http.services.genus_species-blast.loadbalancer.server.port=80"
+ restart_policy:
+ condition: on-failure
+ delay: 5s
+ max_attempts: 3
+ window: 120s
+
+ blast-db:
+# image: postgres:9.6-alpine
+ image: postgres:9.5
+ environment:
+ - POSTGRES_PASSWORD=postgres
+ - PGDATA=/var/lib/postgresql/data/
+ volumes:
+ - ./docker_data/blast_db/:/var/lib/postgresql/data/
+ networks:
+ - genus_species
+
+# wiki:
+# image: quay.io/abretaud/mediawiki
+# environment:
+# MEDIAWIKI_SERVER: http://localhost
+# MEDIAWIKI_PROXY_PREFIX: /sp/genus_species/wiki
+# MEDIAWIKI_SITENAME: Genus species
+# MEDIAWIKI_SECRET_KEY: XXXXXXXXXX
+# MEDIAWIKI_DB_HOST: wiki-db.genus_species
+# MEDIAWIKI_DB_PASSWORD: password
+# MEDIAWIKI_ADMIN_USER: abretaud # ldap user
+# depends_on:
+# - wiki-db
+# volumes:
+# - ./docker_data/wiki_uploads:/images
+# #- ../bipaa_wiki.png:/var/www/mediawiki/resources/assets/wiki.png:ro # To change the logo at the top left
+# networks:
+# - traefikbig
+# - genus_species
+# deploy:
+# labels:
+# - "traefik.http.routers.genus_species-blast.rule=(Host(`localhost`) && PathPrefix(`/sp/genus_species/blast`))"
+# - "traefik.http.routers.genus_species-blast.tls=true"
+# - "traefik.http.routers.genus_species-blast.entryPoints=webs"
+# - "traefik.http.routers.genus_species-blast.middlewares=sp-big-req,sp-auth,sp-app-trailslash,sp-app-prefix"
+# - "traefik.http.services.genus_species-blast.loadbalancer.server.port=80"
+# restart_policy:
+# condition: on-failure
+# delay: 5s
+# max_attempts: 3
+# window: 120s
+
+# wiki-db:
+# image: postgres:9.6-alpine
+# volumes:
+# - ./docker_data/wiki_db/:/var/lib/postgresql/data/
+# networks:
+# - genus_species
+
+networks:
+ traefikbig:
+ external: true
+ genus_species:
+ driver: overlay
+ name: genus_species
diff --git a/templates/traefik_compose_template.yml.j2 b/templates/traefik_compose_template.yml.j2
index 49fe94b..5a70edf 100644
--- a/templates/traefik_compose_template.yml.j2
+++ b/templates/traefik_compose_template.yml.j2
@@ -89,7 +89,7 @@ services:
- ./authelia/:/etc/authelia/:ro
deploy:
labels:
- - "traefik.http.routers.authelia.rule=Host(`{{ auth_hostname }}`)"
+ - "traefik.http.routers.authelia.rule=Host(`{{ authentication_domain_name }}`)"
- "traefik.http.services.authelia.loadbalancer.server.port=9091"
restart_policy:
condition: on-failure
--
GitLab