diff --git a/docker_compose_generator.py b/docker_compose_generator.py
index 33da994515e080f2473ebc9a6e457736bf5e61e3..92f8cb94f1c9cdd5f5eff7055ac171d988c4d49d 100755
--- a/docker_compose_generator.py
+++ b/docker_compose_generator.py
@@ -1,3 +1,6 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
import os
import argparse
import logging
diff --git a/gga_init.py b/gga_init.py
index de88c87e543189c1e016c052bf1d0a34ff7ef5d3..2ed11ef8ea9508733c91ddd7fb306d83fdd69e91 100644
--- a/gga_init.py
+++ b/gga_init.py
@@ -162,14 +162,15 @@ class DeploySpeciesStack(speciesData.SpeciesData):
with open("./docker-compose.yml", 'w') as outfile:
outfile.truncate(0) # Delete file content
for line in organism_content: # Replace env variables by those in the config file
- for env_variable, value in self.config.items():
+ for env_variable, value in self.config.items(): # env variables are stored in this dict
+ # print("ENV VARIABLE: " + env_variable + "\t VALUE: " + value)
if env_variable in line:
line = line.replace(env_variable, value)
- print(line)
break
+ # Write the new line in the docker-compose
outfile.write(line)
- # Store and open the formatted docker-compose.yml file
+ # Create mounts for the current docker-compose
self.create_mounts(working_dir=self.species_dir)
# TODO: obsolete?
@@ -177,8 +178,6 @@ class DeploySpeciesStack(speciesData.SpeciesData):
# subprocess.call(["python3", self.script_dir + "/create_mounts.py"], cwd=self.species_dir,
# stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) # Create mounts for the containers
- print(self.config)
-
# Store the traefik directory path to be able to create volumes for the traefik containers
traefik_dir = None
try:
@@ -205,13 +204,17 @@ class DeploySpeciesStack(speciesData.SpeciesData):
logging.debug("Traefik compose file already exists: %s" % os.path.abspath("./traefik/docker-compose.yml"))
traefik_dir = os.path.abspath(os.path.join(self.main_dir, "traefik"))
+
+ # Create the mounts for the traefik+authelia containers
self.create_mounts(working_dir=traefik_dir)
# subprocess.call(["python3", self.script_dir + "/create_mounts.py"], cwd=self.species_dir) # TODO: obsolete?
os.chdir(self.main_dir)
+
def create_mounts(self, working_dir):
"""
+ Create the folders (volumes) required by a container (to see required volumes, check their compose file)
:return:
"""
@@ -328,10 +331,11 @@ if __name__ == "__main__":
else:
args.config = os.path.abspath(args.config)
+ main_dir = None
if not args.main_directory:
- args.main_directory = os.getcwd()
+ main_dir = os.getcwd()
else:
- args.main_directory = os.path.abspath(args.main_directory)
+ main_dir = os.path.abspath(args.main_directory)
sp_dict_list = utilities.parse_input(os.path.abspath(args.input))
@@ -342,13 +346,14 @@ if __name__ == "__main__":
deploy_stack_for_current_organism = DeploySpeciesStack(parameters_dictionary=sp_dict)
# Setting some of the instance attributes
- deploy_stack_for_current_organism.main_dir = os.getcwd()
+ deploy_stack_for_current_organism.main_dir = main_dir
deploy_stack_for_current_organism.species_dir = os.path.join(deploy_stack_for_current_organism.main_dir,
deploy_stack_for_current_organism.genus_species +
"/")
# Parse the config yaml file
deploy_stack_for_current_organism.config = utilities.parse_config(args.config)
+
# Set the instance url attribute
for env_variable, value in deploy_stack_for_current_organism.config.items():
if env_variable == "custom_host":
@@ -356,7 +361,12 @@ if __name__ == "__main__":
deploy_stack_for_current_organism.genus_lowercase + \
"_" + deploy_stack_for_current_organism.species + \
"/galaxy/"
-
+ break
+ else:
+ deploy_stack_for_current_organism.instance_url = "http://localhost:8888/sp/{0}_{1}/galaxy/".format(
+ deploy_stack_for_current_organism.genus_lowercase,
+ deploy_stack_for_current_organism.species)
+
# Starting
logging.info("gga_init.py called for %s" % deploy_stack_for_current_organism.full_name)
diff --git a/gga_load_data.py b/gga_load_data.py
index 6195f203799b25e48c445fa3ec2d8d4fb4c08072..f3eb17e7cc27131aa1757200580e41146a64ab94 100644
--- a/gga_load_data.py
+++ b/gga_load_data.py
@@ -79,7 +79,7 @@ class LoadData(speciesData.SpeciesData):
for f in os.listdir(d):
if "proteins" in f:
proteins_file = os.path.join(d, f)
- proteins_outfile = os.path.join(d, "oufile_proteins.fa")
+ proteins_outfile = os.path.join(d, "outfile_proteins.fa")
annotation_dir = os.path.abspath(d)
# Formatting the headers
if proteins_file is not None:
@@ -88,9 +88,10 @@ class LoadData(speciesData.SpeciesData):
pattern="^>mRNA",
repl=">protein")
if os.path.exists(annotation_dir + "/outfile_proteins.fa"):
- subprocess.run(["mv", annotation_dir + "/outfile_proteins.fa", f],
+ subprocess.run(["mv", annotation_dir + "/outfile_proteins.fa", proteins_file],
stdout=subprocess.PIPE,
cwd=annotation_dir)
+ subprocess.run(["rm", annotation_dir + "/outfile_proteins.fa"], stdout=subprocess.PIPE, cwd=annotation_dir)
else:
logging.warning("Skipping proteins fasta headers formatting (FileNotFound)")
@@ -492,20 +493,20 @@ class LoadData(speciesData.SpeciesData):
# Wait for uploads to complete
logging.info("Waiting for import jobs to finish... please wait")
- # Checking job state
- while True:
- try:
- # "C" state means the job is completed, no need to wait for it
- ret = subprocess.check_output("squeue | grep -v \"C debug\" | grep -v \"JOBID\" || true",
- shell=True)
- if not len(ret):
- break
- time.sleep(3)
- except subprocess.CalledProcessError as inst:
- if inst.returncode == 153: # queue is empty
- break
- else:
- raise
+ # Checking job state (only necessary if ran using SLURM)
+ # while True:
+ # try:
+ # # "C" state means the job is completed, no need to wait for it
+ # ret = subprocess.check_output("squeue | grep -v \"C debug\" | grep -v \"JOBID\" || true",
+ # shell=True)
+ # if not len(ret):
+ # break
+ # time.sleep(3)
+ # except subprocess.CalledProcessError as inst:
+ # if inst.returncode == 153: # queue is empty
+ # break
+ # else:
+ # raise
time.sleep(10)
@@ -662,9 +663,14 @@ if __name__ == "__main__":
# Set the instance url attribute
for env_variable, value in load_data_for_current_species.config.items():
if env_variable == "custom_host":
- load_data_for_current_species.instance_url = value + load_data_for_current_species.genus_lowercase + \
- "_" + load_data_for_current_species.species + "/galaxy/"
-
+ load_data_for_current_species.instance_url = "http://{0}:8888/sp/{1}_{2}/galaxy/".format(
+ value, load_data_for_current_species.genus_lowercase, load_data_for_current_species.species)
+ break
+ else:
+ load_data_for_current_species.instance_url = "http://localhost:8888/sp/{0}_{1}/galaxy/".format(
+ load_data_for_current_species.genus_lowercase,
+ load_data_for_current_species.species)
+
# Change serexec permissions in repo
try:
os.chmod("%s/serexec" % load_data_for_current_species.script_dir, 0o0777)
@@ -682,20 +688,20 @@ if __name__ == "__main__":
# Testing connection to the instance
load_data_for_current_species.connect_to_instance()
- # # Retrieve datasets
- # logging.info("Finding and copying datasets for %s" % load_data_for_current_species.full_name)
- # load_data_for_current_species.get_source_data_files_from_path()
- # logging.info("Sucessfully copied datasets for %s" % load_data_for_current_species.full_name)
- #
- # # # Format fasta headers (proteins)
- # logging.info("Formatting fasta files headers %s " % load_data_for_current_species.full_name)
- # load_data_for_current_species.batch_modify_fasta_headers()
- # logging.info("Successfully formatted files headers %s " % load_data_for_current_species.full_name)
- #
- # # Load the datasets into a galaxy library
- # logging.info("Setting up library for %s" % load_data_for_current_species.full_name)
- # load_data_for_current_species.setup_library()
- # logging.info("Successfully set up library in galaxy for %s" % load_data_for_current_species.full_name)
+ # Retrieve datasets
+ logging.info("Finding and copying datasets for %s" % load_data_for_current_species.full_name)
+ load_data_for_current_species.get_source_data_files_from_path()
+ logging.info("Sucessfully copied datasets for %s" % load_data_for_current_species.full_name)
+
+ # # Format fasta headers (proteins)
+ logging.info("Formatting fasta files headers %s " % load_data_for_current_species.full_name)
+ load_data_for_current_species.batch_modify_fasta_headers()
+ logging.info("Successfully formatted files headers %s " % load_data_for_current_species.full_name)
+
+ # Load the datasets into a galaxy library
+ logging.info("Setting up library for %s" % load_data_for_current_species.full_name)
+ load_data_for_current_species.setup_library()
+ logging.info("Successfully set up library in galaxy for %s" % load_data_for_current_species.full_name)
# # Set or get the history for the current organism
# load_data_for_current_species.set_get_history()
diff --git a/serexec b/serexec
old mode 100644
new mode 100755
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..7a38c06c22d13eef5f39a0d5844875639af8d902
--- a/serexec
+++ b/serexec
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+SERVICE_NAME=$1; shift
+
+TASK_ID=$(docker service ps --filter 'desired-state=running' $SERVICE_NAME -q)
+#we have only one node
+#NODE_ID=$(docker inspect --format '{{ .NodeID }}' $TASK_ID)
+CONTAINER_ID=$(docker inspect --format '{{ .Status.ContainerStatus.ContainerID }}' $TASK_ID)
+#we have only one node
+#NODE_HOST=$(docker node inspect --format '{{ .Description.Hostname }}' $NODE_ID)
+#we have only one node
+#export DOCKER_HOST="ssh://$USER@$NODE_HOST"
+docker exec -it $CONTAINER_ID "$@"
diff --git a/templates/compose_template.yml b/templates/compose_template.yml
index 016ba9ea42b262977e9e5cf5a5ce9e9809ff6605..f56eac8504ee0588666071125f10e7181897b7cd 100644
--- a/templates/compose_template.yml
+++ b/templates/compose_template.yml
@@ -57,7 +57,7 @@ services:
ENABLE_ORTHOLOGY: 0
ENABLE_ORTHOLOGY_LINKS: 0
THEME: "custom_theme" # Use this to use another theme
- THEME_GIT_CLONE: "custom_theme_git_clone" # Use this to install another theme
+ THEME_GIT_CLONE: "custom_git_clone" # Use this to install another theme
ADMIN_PASSWORD: custom_tripal_admin_password # You need to define it and update it in galaxy config below
networks:
- traefikbig