Skip to content
Snippets Groups Projects
Commit 04d0985d authored by Arthur Le Bars's avatar Arthur Le Bars
Browse files

added docker_compose_generator.py, both as standalone and integrated to autoload.py

parent 67ce6973
No related branches found
No related tags found
1 merge request!1Release 1.0
Hide whitespace changes
Inline Side-by-side
README.md
+ 6
− 2
View file @ 04d0985d
# gga_load_data
Bioblend-based script to load data into GGA
Automated integration of new organisms into GGA instances
TODO: Metadata in /projet/sbr/phaeoexplorer/
\ No newline at end of file
Requirements:
- bioblend (v0.13)
- PyYaml
- pandas
autoload.py
+ 17
− 7
View file @ 04d0985d
......@@ -11,7 +11,7 @@ import numpy
import pandas
import logging
import re
from . import table_parser
from . import table_parser, docker_compose_generator
class Autoload:
......@@ -23,7 +23,7 @@ class Autoload:
To run the workflows, place them in the same directory as this script, and add the method + the workflow
parameters in the main invocation (at the end of the file)
TODO: store workflow as string in
TODO: store main workflow as string
"""
def __init__(self, species_parameters_dictionary: dict, args):
......@@ -55,6 +55,7 @@ class Autoload:
self.datasets = dict()
self.source_files = dict()
self.workflow_name = None
self.docker_compose_generator = None
# Test the connection to the galaxy instance for the current species
# Additionally set some class attributes
......@@ -418,13 +419,12 @@ class Autoload:
return None
if __name__ == "main":
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Automatic loading and interaction with galaxy instance (GGA)"
", following the protocol @ "
"http://gitlab.sb-roscoff.fr/abims/e-infra/gga")
parser.add_argument("-j", "--json", type=str, help="Input JSON file, alternative"
" to input csv file", action="store_true")
parser.add_argument("table", type=str, help="Input table (tabulated file that describes all data)",
parser.add_argument("input", type=str, help="Input table (tabulated file that describes all data)"
"or json file",
action="store_true")
parser.add_argument("-v", "--verbose",
help="Increase output verbosity",
......@@ -440,6 +440,9 @@ if __name__ == "main":
"index tripal data, populate materialized view, "
"create a jbrowse for the current genus_species_strain_sex and add organism to jbrowse",
action="store_true")
parser.add_argument("--generate-docker-compose",
help="Generate docker-compose.yml for current species",
action="store_true")
args = parser.parse_args()
if args.verbose:
......@@ -447,8 +450,15 @@ if __name__ == "main":
else:
logging.basicConfig(level=logging.INFO)
if str(args.input).endswith(".json"):
input_json = args.input
else:
tp = table_parser.TableParser()
logging.info("parsing input table")
tp.table = args.input
input_json = tp.parse_table(mode="simple", method="table_to_json")
sp_dict_list = list()
with open(args.json, 'r') as infile:
with open(input_json, 'r') as infile:
json_sp_dict = json.load(infile)
json_sp_dump = json.dumps(json_sp_dict, indent=4, sort_keys=True)
for json_sp in json_sp_dict:
......
docker-compose.yml 0 → 100644
+ 285
− 0
View file @ 04d0985d
# ./docker_data is created and filled with persistent data that should be backuped
version: '3.7'
services:
proxy:
image: quay.io/abretaud/nginx-ldap:latest
volumes:
- ./src_data/:/project_data/
#- /groups/XXX/:/groups/XXX/:ro # We do this when we have symlinks in src_data pointing to /groups/XXX/...
- ./nginx/conf:/etc/nginx/conf.d
networks:
- traefik
- genus_species
deploy:
labels:
# Download page
- "traefik.http.routers.genus_species-nginx.rule=(Host(`localhost`) && PathPrefix(`/sp/genus_species/download`))"
- "traefik.http.routers.genus_species-nginx.tls=true"
- "traefik.http.routers.genus_species-nginx.entryPoints=webs"
- "traefik.http.routers.genus_species-nginx.middlewares=sp-auth,sp-app-trailslash,sp-prefix"
- "traefik.http.services.genus_species-nginx.loadbalancer.server.port=80"
restart_policy:
condition: on-failure
delay: 5s
max_attempts: 3
window: 120s
tripal:
image: quay.io/galaxy-genome-annotation/tripal:v2.x
depends_on:
- tripal-db
- elasticsearch
volumes:
- ./docker_data/galaxy/:/export/:ro
- ./src_data/:/project_data/:ro
- ./src_data:/data:ro
#- /groups/XXX/:/groups/XXX/:ro # We do this when we have symlinks in src_data pointing to /groups/XXX/...
environment:
DB_HOST: tripal-db.genus_species
BASE_URL_PATH: /sp/genus_species
UPLOAD_LIMIT: 20M
MEMORY_LIMIT: 512M
TRIPAL_GIT_CLONE_MODULES: "https://github.com/abretaud/tripal_rest_api.git[@c6f9021ea5d4c6d7c67c5bd363a7dd9359228bbc] https://github.com/tripal/tripal_elasticsearch.git[@dc7f276046e394a80a7dfc9404cf1a149006eb2a] https://github.com/tripal/tripal_analysis_interpro.git https://github.com/tripal/tripal_analysis_go.git https://github.com/tripal/tripal_analysis_blast.git https://github.com/tripal/tripal_analysis_expression.git[@7240039fdeb4579afd06bbcb989cb7795bd4c342]"
TRIPAL_DOWNLOAD_MODULES: ""
TRIPAL_ENABLE_MODULES: "tripal_analysis_blast tripal_analysis_interpro tripal_analysis_go tripal_rest_api tripal_elasticsearch"
SITE_NAME: "Genus species"
ELASTICSEARCH_HOST: elasticsearch.genus_species
ENABLE_JBROWSE: /jbrowse/?data=data/gspecies
ENABLE_APOLLO: https://localhost/apollo/
ENABLE_BLAST: 1
ENABLE_DOWNLOAD: 1
ENABLE_WIKI: 1
ENABLE_GO: /organism/Genus/species?pane=GO
ENABLE_ORTHOLOGY: 0
ENABLE_ORTHOLOGY_LINKS: http://localhost/sp/orthology/
#THEME: "bipaa" # Use this to use another theme
#THEME_GIT_CLONE: "https://gitlab.inria.fr/abretaud/tripal_bipaa.git" # Use this to install another theme
ADMIN_PASSWORD: XXXXXX # You need to define it and update it in galaxy config below
networks:
- traefik
- genus_species
deploy:
labels:
- "traefik.http.routers.genus_species-tripal.rule=(Host(`localhost`) && PathPrefix(`/sp/genus_species`))"
- "traefik.http.routers.genus_species-tripal.tls=true"
- "traefik.http.routers.genus_species-tripal.entryPoints=webs"
- "traefik.http.routers.genus_species-tripal.middlewares=sp-auth,sp-trailslash,sp-prefix,tripal-addprefix"
- "traefik.http.services.genus_species-tripal.loadbalancer.server.port=80"
restart_policy:
condition: on-failure
delay: 5s
max_attempts: 3
window: 120s
tripal-db:
image: quay.io/galaxy-genome-annotation/chado:1.31-jenkins26-pg9.5
environment:
- POSTGRES_PASSWORD=postgres
# The default chado image would try to install the schema on first run,
# we just want the tools to be available.
- INSTALL_CHADO_SCHEMA=0
volumes:
- ./docker_data/tripal_db/:/var/lib/postgresql/data/
networks:
- genus_species
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:6.6.1
#deploy:
#resources:
#limits:
#memory: 500M
volumes:
- ./docker_data/elastic_search_index/:/usr/share/elasticsearch/data/
environment:
bootstrap.memory_lock: "true"
xpack.security.enabled: "false"
xpack.monitoring.enabled: "false"
xpack.ml.enabled: "false"
xpack.graph.enabled: "false"
xpack.watcher.enabled: "false"
cluster.routing.allocation.disk.threshold_enabled: "false"
ES_JAVA_OPTS: "-Xms500m -Xmx500m"
TAKE_FILE_OWNERSHIP: "true"
networks:
- genus_species
galaxy:
image: quay.io/galaxy-genome-annotation/docker-galaxy-annotation:gmod
volumes:
- ../galaxy_data_libs_SI.py:/opt/setup_data_libraries.py
- ./docker_data/galaxy/:/export/
- ./src_data/:/project_data/:ro
#- /groups/XXX/:/groups/XXX/:ro # We do this when we have symlinks in src_data pointing to /groups/XXX/...
- ./docker_data/jbrowse/:/jbrowse/data/
- ./docker_data/apollo/:/apollo-data/
- ../galaxy_nginx.conf:/etc/nginx/uwsgi_params
environment:
NONUSE: nodejs,proftp,reports,condor
GALAXY_LOGGING: full
GALAXY_CONFIG_BRAND: "Genus species"
GALAXY_CONFIG_ALLOW_LIBRARY_PATH_PASTE: "True"
GALAXY_CONFIG_USE_REMOTE_USER: "True"
GALAXY_CONFIG_REMOTE_USER_MAILDOMAIN: "bipaa"
GALAXY_CONFIG_ADMIN_USERS: "admin@galaxy.org,gogepp@bipaa" # admin@galaxy.org is the default (leave it), gogepp@bipaa is a shared ldap user we use to connect
ENABLE_FIX_PERMS: 0
PROXY_PREFIX: /sp/genus_species/galaxy
GALAXY_TRIPAL_URL: http://tripal.genus_species/tripal/
GALAXY_TRIPAL_PASSWORD: XXXXXX # See tripal config above
GALAXY_WEBAPOLLO_URL: http://one-of-the-swarm-node:8888/apollo/
GALAXY_WEBAPOLLO_USER: "admin_apollo@bipaa"
GALAXY_WEBAPOLLO_PASSWORD: "XXXXXX" # See tripal config below
GALAXY_WEBAPOLLO_EXT_URL: /apollo/
GALAXY_CHADO_DBHOST: tripal-db.genus_species
GALAXY_CHADO_DBSCHEMA: chado
GALAXY_AUTO_UPDATE_DB: 1
GALAXY_AUTO_UPDATE_CONDA: 1
GALAXY_AUTO_UPDATE_TOOLS: "/galaxy-central/tools_1.yaml"
GALAXY_SHARED_DIR: ""
BLAT_ENABLED: 1
networks:
- traefik
- genus_species
deploy:
labels:
- "traefik.http.routers.genus_species-galaxy.rule=(Host(`localhost`) && PathPrefix(`/sp/genus_species/galaxy`))"
- "traefik.http.routers.genus_species-galaxy.tls=true"
- "traefik.http.routers.genus_species-galaxy.entryPoints=webs"
- "traefik.http.routers.genus_species-galaxy.middlewares=sp-auth,sp-app-trailslash,sp-app-prefix"
- "traefik.http.services.genus_species-galaxy.loadbalancer.server.port=80"
restart_policy:
condition: on-failure
delay: 5s
max_attempts: 3
window: 120s
jbrowse:
image: quay.io/galaxy-genome-annotation/jbrowse:v1.16.8
volumes:
- ./docker_data/galaxy/:/export/:ro
- ./src_data/:/project_data/:ro
#- /groups/XXX/:/groups/XXX/:ro # We do this when we have symlinks in src_data pointing to /groups/XXX/...
- ./docker_data/jbrowse/:/jbrowse/data/:ro
networks:
- traefik
- genus_species
deploy:
labels:
- "traefik.http.routers.genus_species-jbrowse.rule=(Host(`localhost`) && PathPrefix(`/sp/genus_species/jbrowse`))"
- "traefik.http.routers.genus_species-jbrowse.tls=true"
- "traefik.http.routers.genus_species-jbrowse.entryPoints=webs"
- "traefik.http.routers.genus_species-jbrowse.middlewares=sp-auth,sp-app-trailslash,sp-app-prefix"
- "traefik.http.services.genus_species-jbrowse.loadbalancer.server.port=80"
restart_policy:
condition: on-failure
delay: 5s
max_attempts: 3
window: 120s
blast:
image: quay.io/abretaud/sf-blast:latest
depends_on:
- blast-db
environment:
DB_HOST: blast-db.genus_species
UPLOAD_LIMIT: 20M
MEMORY_LIMIT: 128M
DB_NAME: 'postgres'
ADMIN_EMAIL: 'xxx@example.org' # email sender
ADMIN_NAME: 'xxxxx' # email sender name
JOBS_METHOD: 'local' # Can be local (= no sge jobs, but run inside the container) or drmaa (= to submit to a cluster)
JOBS_WORK_DIR: '/xxxx/blast_jobs/' # disk accessible both from compute nodes and mounted in this docker (at the same path)
CDD_DELTA_PATH: '/db/cdd_delta/current/flat/cdd_delta'
BLAST_TITLE: 'Genus species blast server'
JOBS_SCHED_NAME: 'blast_gspecies' # job names
PRE_CMD: '. /local/env/envblast-2.6.0.sh; . /local/env/envpython-3.7.1.sh;' # executed at the beginning of each job
APACHE_RUN_USER: 'bipaaweb' # username known by sge
APACHE_RUN_GROUP: 'bipaa' # group known by sge
BASE_URL_PATH: '/sp/genus_species/blast/'
UID: 55914 # username known by sge (for drmaa mode only)
GID: 40259 # group known by sge (for drmaa mode only)
#JOBS_DRMAA_NATIVE: '-p web' # This line and following for slurm
#DRMAA_METHOD: 'slurm' # This line and following for slurm
volumes:
#- ../blast-themes/xxx/:/var/www/blast/app/Resources/:ro # You can theme the app
#- /data1/sge/:/usr/local/sge/:ro # an sge install
#- /xxxx/blast_jobs/:/xxxx/blast_jobs/ # (for drmaa mode only)
- ./blast/banks.yml:/var/www/blast/app/config/banks.yml:ro
- ./blast/links.yml:/etc/blast_links/links.yml:ro
#- /data1/slurm/slurm.conf:/etc/slurm-llnl/slurm.conf:ro # This line and following for slurm
#- /data1/slurm/gres.conf:/etc/slurm-llnl/gres.conf:ro
#- /data1/slurm/cgroup.conf:/etc/slurm-llnl/cgroup.conf:ro
#- /data1/slurm/slurmdbd.conf:/etc/slurm-llnl/slurmdbd.conf:ro
#- /data1/slurm/drmaa/:/etc/slurm-llnl/drmaa/:ro
#- /etc/munge/:/etc/munge/:ro
networks:
- traefik
- genus_species
deploy:
labels:
- "traefik.http.routers.genus_species-blast.rule=(Host(`localhost`) && PathPrefix(`/sp/genus_species/blast`))"
- "traefik.http.routers.genus_species-blast.tls=true"
- "traefik.http.routers.genus_species-blast.entryPoints=webs"
- "traefik.http.routers.genus_species-blast.middlewares=sp-big-req,sp-auth,sp-app-trailslash,sp-app-prefix"
- "traefik.http.services.genus_species-blast.loadbalancer.server.port=80"
restart_policy:
condition: on-failure
delay: 5s
max_attempts: 3
window: 120s
blast-db:
image: postgres:9.6-alpine
environment:
- POSTGRES_PASSWORD=postgres
- PGDATA=/var/lib/postgresql/data/
volumes:
- ./docker_data/blast_db/:/var/lib/postgresql/data/
networks:
- genus_species
wiki:
image: quay.io/abretaud/mediawiki
environment:
MEDIAWIKI_SERVER: http://localhost
MEDIAWIKI_PROXY_PREFIX: /sp/genus_species/wiki
MEDIAWIKI_SITENAME: Genus species
MEDIAWIKI_SECRET_KEY: XXXXXXXXXX
MEDIAWIKI_DB_HOST: wiki-db.genus_species
MEDIAWIKI_DB_PASSWORD: password
MEDIAWIKI_ADMIN_USER: abretaud # ldap user
depends_on:
- wiki-db
volumes:
- ./docker_data/wiki_uploads:/images
#- ../bipaa_wiki.png:/var/www/mediawiki/resources/assets/wiki.png:ro # To change the logo at the top left
networks:
- traefik
- genus_species
deploy:
labels:
- "traefik.http.routers.genus_species-blast.rule=(Host(`localhost`) && PathPrefix(`/sp/genus_species/blast`))"
- "traefik.http.routers.genus_species-blast.tls=true"
- "traefik.http.routers.genus_species-blast.entryPoints=webs"
- "traefik.http.routers.genus_species-blast.middlewares=sp-big-req,sp-auth,sp-app-trailslash,sp-app-prefix"
- "traefik.http.services.genus_species-blast.loadbalancer.server.port=80"
restart_policy:
condition: on-failure
delay: 5s
max_attempts: 3
window: 120s
wiki-db:
image: postgres:9.6-alpine
volumes:
- ./docker_data/wiki_db/:/var/lib/postgresql/data/
networks:
- genus_species
networks:
traefik:
external: true
genus_species:
driver: overlay
name: genus_species
docker_compose_generator.py 0 → 100644
+ 66
− 0
View file @ 04d0985d
import os
import argparse
# import yaml
# import ruamel.yaml
# import json
"""
docker-compose.yml generator
The method "generate" works for both docker-compose architecture (old), or docker stacks (new)
This method will write a formatted docker-compose.yml for the specified organism (only requires genus and species)
Made to work in the integration streamlined script "autoload.py" but can be used as a standalone (either with a CLI
or in another python file as a module)
TODO: write the whole yml dict from scratch (would allow the script to be more reusable into the future and make it
more customizable while being clearer (instead of the default yml string or input docker-compose template)
"""
class DockerComposeGenerator:
def __init__(self):
self.mode = None
self.genus = None
self.species = None
self.template = None
self.outdir = None
def generate(self):
if self.template is None:
self.template = str(os.getcwd() + "/docker-compose.yml")
# default docker-compose if no input template was specified --> doesnt work, yaml doesnt support string replacement as is (https://stackoverflow.com/questions/5484016/how-can-i-do-string-concatenation-or-string-replacement-in-yaml)
# self.template = "{'version': '3.7', 'services': {'proxy': {'image': 'quay.io/abretaud/nginx-ldap:latest', 'volumes': ['./src_data/:/project_data/', './nginx/conf:/etc/nginx/conf.d'], 'networks': ['traefik', 'genus_species'], 'deploy': {'labels': ['traefik.http.routers.genus_species-nginx.rule=(Host(`localhost`) && PathPrefix(`/sp/genus_species/download`))', 'traefik.http.routers.genus_species-nginx.tls=true', 'traefik.http.routers.genus_species-nginx.entryPoints=webs', 'traefik.http.routers.genus_species-nginx.middlewares=sp-auth,sp-app-trailslash,sp-prefix', 'traefik.http.services.genus_species-nginx.loadbalancer.server.port=80'], 'restart_policy': {'condition': 'on-failure', 'delay': '5s', 'max_attempts': 3, 'window': '120s'}}}, 'tripal': {'image': 'quay.io/galaxy-genome-annotation/tripal:v2.x', 'depends_on': ['tripal-db', 'elasticsearch'], 'volumes': ['./docker_data/galaxy/:/export/:ro', './src_data/:/project_data/:ro', './src_data:/data:ro'], 'environment': {'DB_HOST': 'tripal-db.genus_species', 'BASE_URL_PATH': '/sp/genus_species', 'UPLOAD_LIMIT': '20M', 'MEMORY_LIMIT': '512M', 'TRIPAL_GIT_CLONE_MODULES': 'https://github.com/abretaud/tripal_rest_api.git[@c6f9021ea5d4c6d7c67c5bd363a7dd9359228bbc] https://github.com/tripal/tripal_elasticsearch.git[@dc7f276046e394a80a7dfc9404cf1a149006eb2a] https://github.com/tripal/tripal_analysis_interpro.git https://github.com/tripal/tripal_analysis_go.git https://github.com/tripal/tripal_analysis_blast.git https://github.com/tripal/tripal_analysis_expression.git[@7240039fdeb4579afd06bbcb989cb7795bd4c342]', 'TRIPAL_DOWNLOAD_MODULES': '', 'TRIPAL_ENABLE_MODULES': 'tripal_analysis_blast tripal_analysis_interpro tripal_analysis_go tripal_rest_api tripal_elasticsearch', 'SITE_NAME': 'Genus species', 'ELASTICSEARCH_HOST': 'elasticsearch.genus_species', 'ENABLE_JBROWSE': '/jbrowse/?data=data/gspecies', 'ENABLE_APOLLO': 'https://localhost/apollo/', 'ENABLE_BLAST': 1, 'ENABLE_DOWNLOAD': 1, 'ENABLE_WIKI': 1, 'ENABLE_GO': '/organism/Genus/species?pane=GO', 'ENABLE_ORTHOLOGY': 0, 'ENABLE_ORTHOLOGY_LINKS': 'http://localhost/sp/orthology/', 'ADMIN_PASSWORD': 'XXXXXX'}, 'networks': ['traefik', 'genus_species'], 'deploy': {'labels': ['traefik.http.routers.genus_species-tripal.rule=(Host(`localhost`) && PathPrefix(`/sp/genus_species`))', 'traefik.http.routers.genus_species-tripal.tls=true', 'traefik.http.routers.genus_species-tripal.entryPoints=webs', 'traefik.http.routers.genus_species-tripal.middlewares=sp-auth,sp-trailslash,sp-prefix,tripal-addprefix', 'traefik.http.services.genus_species-tripal.loadbalancer.server.port=80'], 'restart_policy': {'condition': 'on-failure', 'delay': '5s', 'max_attempts': 3, 'window': '120s'}}}, 'tripal-db': {'image': 'quay.io/galaxy-genome-annotation/chado:1.31-jenkins26-pg9.5', 'environment': ['POSTGRES_PASSWORD=postgres', 'INSTALL_CHADO_SCHEMA=0'], 'volumes': ['./docker_data/tripal_db/:/var/lib/postgresql/data/'], 'networks': ['genus_species']}, 'elasticsearch': {'image': 'docker.elastic.co/elasticsearch/elasticsearch:6.6.1', 'volumes': ['./docker_data/elastic_search_index/:/usr/share/elasticsearch/data/'], 'environment': {'bootstrap.memory_lock': 'true', 'xpack.security.enabled': 'false', 'xpack.monitoring.enabled': 'false', 'xpack.ml.enabled': 'false', 'xpack.graph.enabled': 'false', 'xpack.watcher.enabled': 'false', 'cluster.routing.allocation.disk.threshold_enabled': 'false', 'ES_JAVA_OPTS': '-Xms500m -Xmx500m', 'TAKE_FILE_OWNERSHIP': 'true'}, 'networks': ['genus_species']}, 'galaxy': {'image': 'quay.io/galaxy-genome-annotation/docker-galaxy-annotation:gmod', 'volumes': ['../galaxy_data_libs_SI.py:/opt/setup_data_libraries.py', './docker_data/galaxy/:/export/', './src_data/:/project_data/:ro', './docker_data/jbrowse/:/jbrowse/data/', './docker_data/apollo/:/apollo-data/', '../galaxy_nginx.conf:/etc/nginx/uwsgi_params'], 'environment': {'NONUSE': 'nodejs,proftp,reports,condor', 'GALAXY_LOGGING': 'full', 'GALAXY_CONFIG_BRAND': 'Genus species', 'GALAXY_CONFIG_ALLOW_LIBRARY_PATH_PASTE': 'True', 'GALAXY_CONFIG_USE_REMOTE_USER': 'True', 'GALAXY_CONFIG_REMOTE_USER_MAILDOMAIN': 'bipaa', 'GALAXY_CONFIG_ADMIN_USERS': 'admin@galaxy.org,gogepp@bipaa', 'ENABLE_FIX_PERMS': 0, 'PROXY_PREFIX': '/sp/genus_species/galaxy', 'GALAXY_TRIPAL_URL': 'http://tripal.genus_species/tripal/', 'GALAXY_TRIPAL_PASSWORD': 'XXXXXX', 'GALAXY_WEBAPOLLO_URL': 'http://one-of-the-swarm-node:8888/apollo/', 'GALAXY_WEBAPOLLO_USER': 'admin_apollo@bipaa', 'GALAXY_WEBAPOLLO_PASSWORD': 'XXXXXX', 'GALAXY_WEBAPOLLO_EXT_URL': '/apollo/', 'GALAXY_CHADO_DBHOST': 'tripal-db.genus_species', 'GALAXY_CHADO_DBSCHEMA': 'chado', 'GALAXY_AUTO_UPDATE_DB': 1, 'GALAXY_AUTO_UPDATE_CONDA': 1, 'GALAXY_AUTO_UPDATE_TOOLS': '/galaxy-central/tools_1.yaml', 'GALAXY_SHARED_DIR': '', 'BLAT_ENABLED': 1}, 'networks': ['traefik', 'genus_species'], 'deploy': {'labels': ['traefik.http.routers.genus_species-galaxy.rule=(Host(`localhost`) && PathPrefix(`/sp/genus_species/galaxy`))', 'traefik.http.routers.genus_species-galaxy.tls=true', 'traefik.http.routers.genus_species-galaxy.entryPoints=webs', 'traefik.http.routers.genus_species-galaxy.middlewares=sp-auth,sp-app-trailslash,sp-app-prefix', 'traefik.http.services.genus_species-galaxy.loadbalancer.server.port=80'], 'restart_policy': {'condition': 'on-failure', 'delay': '5s', 'max_attempts': 3, 'window': '120s'}}}, 'jbrowse': {'image': 'quay.io/galaxy-genome-annotation/jbrowse:v1.16.8', 'volumes': ['./docker_data/galaxy/:/export/:ro', './src_data/:/project_data/:ro', './docker_data/jbrowse/:/jbrowse/data/:ro'], 'networks': ['traefik', 'genus_species'], 'deploy': {'labels': ['traefik.http.routers.genus_species-jbrowse.rule=(Host(`localhost`) && PathPrefix(`/sp/genus_species/jbrowse`))', 'traefik.http.routers.genus_species-jbrowse.tls=true', 'traefik.http.routers.genus_species-jbrowse.entryPoints=webs', 'traefik.http.routers.genus_species-jbrowse.middlewares=sp-auth,sp-app-trailslash,sp-app-prefix', 'traefik.http.services.genus_species-jbrowse.loadbalancer.server.port=80'], 'restart_policy': {'condition': 'on-failure', 'delay': '5s', 'max_attempts': 3, 'window': '120s'}}}, 'blast': {'image': 'quay.io/abretaud/sf-blast:latest', 'depends_on': ['blast-db'], 'environment': {'DB_HOST': 'blast-db.genus_species', 'UPLOAD_LIMIT': '20M', 'MEMORY_LIMIT': '128M', 'DB_NAME': 'postgres', 'ADMIN_EMAIL': 'xxx@example.org', 'ADMIN_NAME': 'xxxxx', 'JOBS_METHOD': 'local', 'JOBS_WORK_DIR': '/xxxx/blast_jobs/', 'CDD_DELTA_PATH': '/db/cdd_delta/current/flat/cdd_delta', 'BLAST_TITLE': 'Genus species blast server', 'JOBS_SCHED_NAME': 'blast_gspecies', 'PRE_CMD': '. /local/env/envblast-2.6.0.sh; . /local/env/envpython-3.7.1.sh;', 'APACHE_RUN_USER': 'bipaaweb', 'APACHE_RUN_GROUP': 'bipaa', 'BASE_URL_PATH': '/sp/genus_species/blast/', 'UID': 55914, 'GID': 40259}, 'volumes': ['./blast/banks.yml:/var/www/blast/app/config/banks.yml:ro', './blast/links.yml:/etc/blast_links/links.yml:ro'], 'networks': ['traefik', 'genus_species'], 'deploy': {'labels': ['traefik.http.routers.genus_species-blast.rule=(Host(`localhost`) && PathPrefix(`/sp/genus_species/blast`))', 'traefik.http.routers.genus_species-blast.tls=true', 'traefik.http.routers.genus_species-blast.entryPoints=webs', 'traefik.http.routers.genus_species-blast.middlewares=sp-big-req,sp-auth,sp-app-trailslash,sp-app-prefix', 'traefik.http.services.genus_species-blast.loadbalancer.server.port=80'], 'restart_policy': {'condition': 'on-failure', 'delay': '5s', 'max_attempts': 3, 'window': '120s'}}}, 'blast-db': {'image': 'postgres:9.6-alpine', 'environment': ['POSTGRES_PASSWORD=postgres', 'PGDATA=/var/lib/postgresql/data/'], 'volumes': ['./docker_data/blast_db/:/var/lib/postgresql/data/'], 'networks': ['genus_species']}, 'wiki': {'image': 'quay.io/abretaud/mediawiki', 'environment': {'MEDIAWIKI_SERVER': 'http://localhost', 'MEDIAWIKI_PROXY_PREFIX': '/sp/genus_species/wiki', 'MEDIAWIKI_SITENAME': 'Genus species', 'MEDIAWIKI_SECRET_KEY': 'XXXXXXXXXX', 'MEDIAWIKI_DB_HOST': 'wiki-db.genus_species', 'MEDIAWIKI_DB_PASSWORD': 'password', 'MEDIAWIKI_ADMIN_USER': 'abretaud'}, 'depends_on': ['wiki-db'], 'volumes': ['./docker_data/wiki_uploads:/images'], 'networks': ['traefik', 'genus_species'], 'deploy': {'labels': ['traefik.http.routers.genus_species-blast.rule=(Host(`localhost`) && PathPrefix(`/sp/genus_species/blast`))', 'traefik.http.routers.genus_species-blast.tls=true', 'traefik.http.routers.genus_species-blast.entryPoints=webs', 'traefik.http.routers.genus_species-blast.middlewares=sp-big-req,sp-auth,sp-app-trailslash,sp-app-prefix', 'traefik.http.services.genus_species-blast.loadbalancer.server.port=80'], 'restart_policy': {'condition': 'on-failure', 'delay': '5s', 'max_attempts': 3, 'window': '120s'}}}, 'wiki-db': {'image': 'postgres:9.6-alpine', 'volumes': ['./docker_data/wiki_db/:/var/lib/postgresql/data/'], 'networks': ['genus_species']}}, 'networks': {'traefik': {'external': True}, 'genus_species': {'driver': 'overlay', 'name': 'genus_species'}}}"
#
else:
with open(self.template, 'r') as infile:
content = list()
for line in infile:
content.append(line.replace("genus_species", str(self.genus.lower() + "_" + self.species)).replace("Genus species", str(self.genus + " " + self.species)).replace("Genus/species", str(self.genus + "/" + self.species)).replace("gspecies", str(self.genus.lower()[0] + self.species)))
self.write_yml(content=content)
def write_yml(self, content):
with open(self.outdir + "/docker-compose.yml", 'w') as outfile:
for line in content:
outfile.write(line)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Generator of docker-compose.yml for GGA automated integration "
"following the templates available @ "
"https://gitlab.inria.fr/abretaud/genodock_demo/")
parser.add_argument("-g", "--genus", type=str, help="input genus")
parser.add_argument("-s", "--species", type=str, help="input species")
parser.add_argument("-t", "--template", type=str, help="input template docker-compose.yml (compose or stack), optional")
parser.add_argument("-o", "--outdir", type=str, help="where to write the output docker-compose")
args = parser.parse_args()
generator = DockerComposeGenerator()
generator.genus = args.genus
generator.species = args.species
if args.template:
generator.template = args.template
generator.outdir = args.outdir
generator.generate()
metadata_generator.py 0 → 100644
+ 23
− 0
View file @ 04d0985d
import os
"""
Metadata generator for gga_auto_load
Creates a file that summarizes the organisms added by the script and what was done in the dedicated galaxy instance
This organism metadata file is located in the metadata directory of the organism directory (i.e /genus_species/metadata)
By default, will also update a general metadata file (located in the parent directory i.e where all the organisms
directories are located)
Metadata format: .yml
"""
class MetadataGenerator:
def __init__(self):
self.maindir = None
self.meta = None
self.mainfile = None
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment