From dfe0f00fe55c7a85e7906382dd97c7869fa87c90 Mon Sep 17 00:00:00 2001
From: Arthur Le Bars <arthur.le-bars@sb-roscoff.fr>
Date: Thu, 18 Feb 2021 17:55:45 +0100
Subject: [PATCH] jinja2 templating for authelia and traefik entrypoint and
host, gga_get_data only copies files that exist
---
gga_get_data.py | 8 +-
gga_init.py | 89 +++--
.../authelia_phaeoexplorer_config.yml.j2 | 355 ++++++++++++++++++
3 files changed, 427 insertions(+), 25 deletions(-)
create mode 100644 templates/authelia_phaeoexplorer_config.yml.j2
diff --git a/gga_get_data.py b/gga_get_data.py
index e9eb11e..4d56b30 100644
--- a/gga_get_data.py
+++ b/gga_get_data.py
@@ -82,10 +82,10 @@ class GetData(speciesData.SpeciesData):
annotation_datasets = ["gff_path", "transcripts_path", "proteins_path", "orthofinder_path", "interpro_path", "blastp_path", "blastx_path"]
# Where to store blast results?
- search_excluded_datasets = ["interpro_path", "orthofinder_path", "blastp_path", "blastx_path"]
- # These datasets will not be searched if missing in the input file
+ # search_excluded_datasets = ["interpro_path", "orthofinder_path", "blastp_path", "blastx_path"]
+ # # These datasets will not be searched if missing in the input file
- # Copy dataset in the organism src_data dir tree correct folder
+ # Copy datasets in the organism src_data dir tree correct folder
for k, v in datasets_to_get.items():
if k in genome_datasets:
logging.info("Copying {0} into {1}".format(v, organism_genome_dir))
@@ -93,7 +93,7 @@ class GetData(speciesData.SpeciesData):
shutil.copyfile(os.path.abspath(v), os.path.join(organism_genome_dir, os.path.basename(v)))
except Exception as exc:
logging.warning("Could not copy {1} ({2})".format(v, exc))
- elif k in annotation_datasets:
+ elif k in annotation_datasets:
logging.info("Copying {0} into {1}".format(v, organism_annotation_dir))
try:
shutil.copyfile(os.path.abspath(v), os.path.join(organism_annotation_dir, os.path.basename(v)))
diff --git a/gga_init.py b/gga_init.py
index 9c990af..36b1afc 100644
--- a/gga_init.py
+++ b/gga_init.py
@@ -108,7 +108,7 @@ class DeploySpeciesStack(speciesData.SpeciesData):
logging.info("Directory tree generated for %s" % self.full_name)
- def make_compose_files(self):
+ def make_compose_files(self, entrypoint_mode):
"""
Create a formatted copy of the template compose file inside a species directory tree
@@ -135,6 +135,15 @@ class DeploySpeciesStack(speciesData.SpeciesData):
# Merge the two dicts
render_vars = {**self.config, **input_vars}
+ # Define entrypoint mode (http=web, https=webs)
+ if entrypoint_mode == "web" or entrypoint_mode == "webs":
+ entrypoint = {"entrypoint": entrypoint_mode}
+ else:
+ logging.critical("Entrypoint mode defined is invalid (%s defined, valid options : ['webs', 'web']")
+ sys.exit()
+ # Add entrypoint to render_vars dor templating
+ render_vars = {**render_vars, **entrypoint}
+
# Render the gspecies docker-compose file and write it
gspecies_compose_template = env.get_template("gspecies_compose_template.yml.j2")
gspecies_compose_output = gspecies_compose_template.render(render_vars)
@@ -183,7 +192,7 @@ def make_dirs(dir_paths_li):
return created_dir_paths_li
-def make_traefik_compose_files(config, main_dir):
+def make_traefik_compose_files(config, main_dir, entrypoint_mode):
"""
Create or update the traefik directory, docker-compose file and authelia conf files
Only called when the argument "--traefik" is specified
@@ -196,6 +205,15 @@ def make_traefik_compose_files(config, main_dir):
script_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
render_vars = config
+ # Define entrypoint mode (http=web, https=webs)
+ if entrypoint_mode == "web" or entrypoint_mode == "webs":
+ entrypoint = {"entrypoint": entrypoint_mode}
+ else:
+ logging.critical("Entrypoint mode defined is invalid (%s defined, valid options : ['webs', 'web']" % entrypoint_mode)
+ sys.exit()
+ # Add entrypoint to render_vars dor templating
+ render_vars = {**render_vars, **entrypoint}
+
os.chdir(main_dir)
# Create directory tree
@@ -220,13 +238,15 @@ def make_traefik_compose_files(config, main_dir):
if config["authelia_config_path"]:
if not config["authelia_config_path"] == "" or not config["authelia_config_path"] == "/path/to/authelia/config":
if os.path.isfile(os.path.abspath(config["authelia_config_path"])):
- try:
- shutil.copy(os.path.abspath(config["authelia_config_path"]), "./traefik/authelia")
- except Exception as exc:
- logging.critical("Cannot copy custom Authelia config file (%s)" % config["authelia_config_path"])
- sys.exit(exc)
+ authelia_config_template = env.get_template(os.path.basename(config["authelia_config_path"]))
+ authelia_config_output = authelia_config_template.render(render_vars)
+ with open(os.path.join(main_dir, "traefik/authelia/configuration.yml"), 'w') as authelia_config_file:
+ logging.info("Writing authelia configuration.yml")
+ authelia_config_file.truncate(0)
+ authelia_config_file.write(authelia_config_output)
else:
- logging.critical("Custom Authelia config file not found (%s)" % config["authelia_config_path"])
+ logging.critical("Cannot find authelia configuration template path (%s)" % config["authelia_config_path"])
+ sys.exit()
# Path to the authelia users in the repo
authelia_users_path = script_dir + "/templates/authelia_users_template.yml"
@@ -292,7 +312,7 @@ def create_mounts(working_dir, main_dir):
logging.critical("Cannot access %s, exiting" % main_dir)
sys.exit(exc)
-def deploy_stacks(input_list, main_dir):
+def deploy_stacks(input_list, main_dir, deploy_traefik):
"""
This function first deploys/redeploys the traefik stack, then deploys/redeploys the organism stack, then redeploys the traefik stack
This function is executed outside the "main" loop of input species
@@ -306,15 +326,18 @@ def deploy_stacks(input_list, main_dir):
# Get species for which to deploy the stacks
to_deploy_species_li = utilities.get_species_to_deploy(sp_dict_list=input_list)
- # Create the swarm cluster if needed
- subprocess.call(["docker", "swarm", "init"],
- stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=main_dir)
-
- # Deploy traefik stack
- os.chdir("./traefik")
- subprocess.call(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "traefik"],
- stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=".")
- os.chdir(main_dir)
+ if deploy_traefik:
+ # Create the swarm cluster if needed
+ logging.info("Initializing docker swarm (adding node)")
+ subprocess.call(["docker", "swarm", "init"],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=main_dir)
+
+ # Deploy traefik stack
+ logging.info("Deploying traefik stack")
+ os.chdir("./traefik")
+ subprocess.call(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "traefik"],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=".")
+ os.chdir(main_dir)
# Deploy individual species stacks
for sp in to_deploy_species_li:
@@ -325,6 +348,7 @@ def deploy_stacks(input_list, main_dir):
os.chdir(main_dir)
# Update traefik stack
+ logging.info("Re-deploying traefik stack")
os.chdir("./traefik")
subprocess.call(["docker", "stack", "deploy", "-c", "./docker-compose.yml", "traefik"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=".")
@@ -372,6 +396,14 @@ if __name__ == "__main__":
help="Initialize/wverwrite traefik directory all docker-compose and conf files in the traefik and authelia directories (default=False)",
action="store_true")
+ parser.add_argument("--http",
+ help="Use a HTTP authelia configuration",
+ action="store_true")
+
+ parser.add_argument("--https",
+ help="Use a HTTPS authelia configuration",
+ action="store_true")
+
args = parser.parse_args()
if args.verbose:
@@ -396,7 +428,13 @@ if __name__ == "__main__":
# Create traefik directory and compose files if specified
if args.traefik:
config = utilities.parse_config(args.config)
- make_traefik_compose_files(config=config, main_dir=main_dir)
+ if args.https:
+ make_traefik_compose_files(config=config, main_dir=main_dir, entrypoint_mode="webs")
+ elif args.http:
+ make_traefik_compose_files(config=config, main_dir=main_dir, entrypoint_mode="web")
+ else:
+ logging.critical("Entrypoint mode is not defined (missing http or https argument")
+ sys.exit()
logging.info("Deploying stacks for organisms in input file %s" % args.input)
for sp_dict in sp_dict_list:
@@ -434,9 +472,18 @@ if __name__ == "__main__":
logging.info("Successfully generated the directory tree for %s" % deploy_stack_for_current_organism.full_name)
# Make compose files
- deploy_stack_for_current_organism.make_compose_files()
+ if args.https:
+ deploy_stack_for_current_organism.make_compose_files(entrypoint_mode="webs")
+ elif args.http:
+ deploy_stack_for_current_organism.make_compose_files(entrypoint_mode="web")
+ else:
+ logging.critical("Entrypoint mode is not defined (missing http or https argument")
+ sys.exit()
logging.info("Successfully generated the docker-compose files for %s" % deploy_stack_for_current_organism.full_name)
logging.info("Deploying stacks")
- deploy_stacks(input_list=sp_dict_list, main_dir=main_dir)
+ if args.traefik:
+ deploy_stacks(input_list=sp_dict_list, main_dir=main_dir, deploy_traefik=True)
+ else:
+ deploy_stacks(input_list=sp_dict_list, main_dir=main_dir, deploy_traefik=False)
logging.info("All stacks deployed for organisms in input file %s" % args.input)
diff --git a/templates/authelia_phaeoexplorer_config.yml.j2 b/templates/authelia_phaeoexplorer_config.yml.j2
new file mode 100644
index 0000000..0b847a2
--- /dev/null
+++ b/templates/authelia_phaeoexplorer_config.yml.j2
@@ -0,0 +1,355 @@
+###############################################################
+# Authelia configuration #
+###############################################################
+
+# The host and port to listen on
+host: 0.0.0.0
+port: 9091
+# tls_key: /var/lib/authelia/ssl/key.pem
+# tls_cert: /var/lib/authelia/ssl/cert.pem
+
+# Level of verbosity for logs: info, debug, trace
+log_level: info
+## File path where the logs will be written. If not set logs are written to stdout.
+# log_file_path: /var/log/authelia
+
+# The secret used to generate JWT tokens when validating user identity by
+# email confirmation.
+# This secret can also be set using the env variables AUTHELIA_JWT_SECRET
+jwt_secret: secret
+
+# Default redirection URL
+#
+# If user tries to authenticate without any referer, Authelia
+# does not know where to redirect the user to at the end of the
+# authentication process.
+# This parameter allows you to specify the default redirection
+# URL Authelia will use in such a case.
+#
+# Note: this parameter is optional. If not provided, user won't
+# be redirected upon successful authentication.
+default_redirection_url: https://{{ hostname }}/
+
+# Google Analytics Tracking ID to track the usage of the portal
+# using a Google Analytics dashboard.
+#
+## google_analytics: UA-00000-01
+
+# TOTP Settings
+#
+# Parameters used for TOTP generation
+#totp:
+ # The issuer name displayed in the Authenticator application of your choice
+ # See: https://github.com/google/google-authenticator/wiki/Key-Uri-Format for more info on issuer names
+ #issuer: authelia.com
+ # The period in seconds a one-time password is current for. Changing this will require all users to register
+ # their TOTP applications again.
+ # Warning: before changing period read the docs link below.
+ #period: 30
+ # The skew controls number of one-time passwords either side of the current one that are valid.
+ # Warning: before changing skew read the docs link below.
+ #skew: 1
+ # See: https://docs.authelia.com/configuration/one-time-password.html#period-and-skew to read the documentation.
+
+# Duo Push API
+#
+# Parameters used to contact the Duo API. Those are generated when you protect an application
+# of type "Partner Auth API" in the management panel.
+#duo_api:
+ #hostname: api-123456789.example.com
+ #integration_key: ABCDEF
+ # This secret can also be set using the env variables AUTHELIA_DUO_API_SECRET_KEY
+ #secret_key: 1234567890abcdefghifjkl
+
+# The authentication backend to use for verifying user passwords
+# and retrieve information such as email address and groups
+# users belong to.
+#
+# There are two supported backends: 'ldap' and 'file'.
+authentication_backend:
+ # Disable both the HTML element and the API for reset password functionality
+ disable_reset_password: true
+
+ # LDAP backend configuration.
+ #
+ # This backend allows Authelia to be scaled to more
+ # than one instance and therefore is recommended for
+ # production.
+# ldap:
+ # The url to the ldap server. Scheme can be ldap:// or ldaps://
+ url: ldap://ldap.sb-roscoff.fr/o=sb-roscoff,c=fr?uid?sub?
+ # Skip verifying the server certificate (to allow self-signed certificate).
+ skip_verify: false
+
+ # The base dn for every entries
+# base_dn: dc=genouest,dc=org
+
+ # The attribute holding the username of the user. This attribute is used to populate
+ # the username in the session information. It was introduced due to #561 to handle case
+ # insensitive search queries.
+ # For you information, Microsoft Active Directory usually uses 'sAMAccountName' and OpenLDAP
+ # usually uses 'uid'
+ username_attribute: uid
+
+ # An additional dn to define the scope to all users
+# additional_users_dn: ou=users
+
+ # The users filter used in search queries to find the user profile based on input filled in login form.
+ # Various placeholders are available to represent the user input and back reference other options of the configuration:
+ # - {input} is a placeholder replaced by what the user inputs in the login form.
+ # - {username_attribute} is a placeholder replaced by what is configured in `username_attribute`.
+ # - {mail_attribute} is a placeholder replaced by what is configured in `mail_attribute`.
+ # - DON'T USE - {0} is an alias for {input} supported for backward compatibility but it will be deprecated in later versions, so please don't use it.
+ #
+ # Recommended settings are as follows:
+ # - Microsoft Active Directory: (&({username_attribute}={input})(objectCategory=person)(objectClass=user))
+ # - OpenLDAP: (&({username_attribute}={input})(objectClass=person))' or '(&({username_attribute}={input})(objectClass=inetOrgPerson))
+ #
+ # To allow sign in both with username and email, one can use a filter like
+ # (&(|({username_attribute}={input})({mail_attribute}={input}))(objectClass=person))
+ users_filter: (&({username_attribute}={input})(objectClass=abimsPerson)(isActive=TRUE))
+
+ # An additional dn to define the scope of groups
+# additional_groups_dn: ou=groups
+
+ # The groups filter used in search queries to find the groups of the user.
+ # - {input} is a placeholder replaced by what the user inputs in the login form.
+ # - {username} is a placeholder replace by the username stored in LDAP (based on `username_attribute`).
+ # - {dn} is a matcher replaced by the user distinguished name, aka, user DN.
+ # - {username_attribute} is a placeholder replaced by what is configured in `username_attribute`.
+ # - {mail_attribute} is a placeholder replaced by what is configured in `mail_attribute`.
+ # - DON'T USE - {0} is an alias for {input} supported for backward compatibility but it will be deprecated in later versions, so please don't use it.
+ # - DON'T USE - {1} is an alias for {username} supported for backward compatibility but it will be deprecated in later version, so please don't use it.
+# groups_filter: (&(member={dn})(objectclass=bipaaGroup))
+
+ # The attribute holding the name of the group
+# group_name_attribute: cn
+
+ # The attribute holding the mail address of the user
+# mail_attribute: mail
+
+# The username and password of the admin user.
+ user: cn=admin,dc=abims,dc=org
+# This secret can also be set using the env variables AUTHELIA_AUTHENTICATION_BACKEND_LDAP_PASSWORD
+ password: dev
+
+ # File backend configuration.
+ #
+ # With this backend, the users database is stored in a file
+ # which is updated when users reset their passwords.
+ # Therefore, this backend is meant to be used in a dev environment
+ # and not in production since it prevents Authelia to be scaled to
+ # more than one instance. The options under password_options have sane
+ # defaults, and as it has security implications it is highly recommended
+ # you leave the default values. Before considering changing these settings
+ # please read the docs page below:
+ # https://docs.authelia.com/configuration/authentication/file.html#password-hash-algorithm-tuning
+ #
+ # file:
+ # path: ./users_database.yml
+ file:
+ path: ./users.yml
+ password_options:
+ algorithm: argon2id
+ iterations: 1
+ key_length: 32
+ salt_length: 16
+ memory: 1024
+ parallelism: 8
+
+
+# Access Control
+#
+# Access control is a list of rules defining the authorizations applied for one
+# resource to users or group of users.
+#
+# If 'access_control' is not defined, ACL rules are disabled and the 'bypass'
+# rule is applied, i.e., access is allowed to anyone. Otherwise restrictions follow
+# the rules defined.
+#
+# Note: One can use the wildcard * to match any subdomain.
+# It must stand at the beginning of the pattern. (example: *.mydomain.com)
+#
+# Note: You must put patterns containing wildcards between simple quotes for the YAML
+# to be syntactically correct.
+#
+# Definition: A 'rule' is an object with the following keys: 'domain', 'subject',
+# 'policy' and 'resources'.
+#
+# - 'domain' defines which domain or set of domains the rule applies to.
+#
+# - 'subject' defines the subject to apply authorizations to. This parameter is
+# optional and matching any user if not provided. If provided, the parameter
+# represents either a user or a group. It should be of the form 'user:<username>'
+# or 'group:<groupname>'.
+#
+# - 'policy' is the policy to apply to resources. It must be either 'bypass',
+# 'one_factor', 'two_factor' or 'deny'.
+#
+# - 'resources' is a list of regular expressions that matches a set of resources to
+# apply the policy to. This parameter is optional and matches any resource if not
+# provided.
+#
+# Note: the order of the rules is important. The first policy matching
+# (domain, resource, subject) applies.
+access_control:
+ # Default policy can either be 'bypass', 'one_factor', 'two_factor' or 'deny'.
+ # It is the policy applied to any resource if there is no policy to be applied
+ # to the user.
+ default_policy: bypass
+
+ rules:
+ # The login portal is freely accessible (redirectino loop otherwise)
+ - domain: auth.example.org
+ policy: bypass
+
+ # Apollo needs to be authenticated
+ - domain: {{ hostname }}
+ resources:
+ - "^/apollo/.*$"
+ policy: allow
+
+ # traefik dashboard is restricted to a group from ldap
+ - domain: {{ hostname }}
+ resources:
+ - "^/traefik/.*$"
+ policy: allow
+ subject: "group:ldap_admin"
+ - domain: allow
+ resources:
+ - "^/traefik/.*$"
+ policy: allow
+
+ # All galaxies are restricted to a group from ldap
+ - domain: {{ hostname }}
+ resources:
+ - "^/sp/.+/galaxy/.*$"
+ policy: one_factor
+ subject: "group:ldap_admin"
+ - domain: {{ hostname }}
+ resources:
+ - "^/sp/.+/galaxy$"
+ policy: allow
+
+ # A genome restricted to an ldap group
+ - domain: {{ hostname }}
+ resources:
+ - "^/sp/genus_species/.*$"
+ policy: one_factor
+ subject: "group:gspecies"
+ - domain: {{ hostname }}
+ resources:
+ - "^/sp/.*$"
+ policy: allow
+
+
+# Configuration of session cookies
+#
+# The session cookies identify the user once logged in.
+session:
+ # The name of the session cookie. (default: authelia_session).
+ name: authelia_abims_session
+
+ # The secret to encrypt the session data. This is only used with Redis.
+ # This secret can also be set using the env variables AUTHELIA_SESSION_SECRET
+ secret: WXXXXXXXXXXXXXXXXXXXcXXXXXXXXXXXXXX
+
+ # The time in seconds before the cookie expires and session is reset.
+ expiration: 3600000 # 1000 hour
+
+ # The inactivity time in seconds before the session is reset.
+ # abretaud: We get an Unauthorized message when reaching this threshold => disabling by making > cookie lifetime
+ inactivity: 3700000 # 5 minutes
+
+ # The remember me duration.
+ # Value of 0 disables remember me.
+ # Value is in seconds, or duration notation. See: https://docs.authelia.com/configuration/index.html#duration-notation-format
+ # Longer periods are considered less secure because a stolen cookie will last longer giving attackers more time to spy
+ # or attack. Currently the default is 1M or 1 month.
+ remember_me_duration: 1M
+
+ # The domain to protect.
+ # Note: the authenticator must also be in that domain. If empty, the cookie
+ # is restricted to the subdomain of the issuer.
+ domain: genouest.org
+
+ # The redis connection details
+ redis:
+ host: authelia-redis
+ port: 6379
+ # This secret can also be set using the env variables AUTHELIA_SESSION_REDIS_PASSWORD
+ #password: authelia
+ # This is the Redis DB Index https://redis.io/commands/select (sometimes referred to as database number, DB, etc).
+ #database_index: 0
+
+# Configuration of the authentication regulation mechanism.
+#
+# This mechanism prevents attackers from brute forcing the first factor.
+# It bans the user if too many attempts are done in a short period of
+# time.
+regulation:
+ # The number of failed login attempts before user is banned.
+ # Set it to 0 to disable regulation.
+ max_retries: 3
+
+ # The time range during which the user can attempt login before being banned.
+ # The user is banned if the authentication failed 'max_retries' times in a 'find_time' seconds window.
+ # Find Time accepts duration notation. See: https://docs.authelia.com/configuration/index.html#duration-notation-format
+ find_time: 2m
+
+ # The length of time before a banned user can login again.
+ # Ban Time accepts duration notation. See: https://docs.authelia.com/configuration/index.html#duration-notation-format
+ ban_time: 5m
+
+# Configuration of the storage backend used to store data and secrets.
+#
+# You must use only an available configuration: local, mysql, postgres
+storage:
+ postgres:
+ host: authelia-db
+ port: 5432
+ database: postgres
+ username: postgres
+ # # This secret can also be set using the env variables AUTHELIA_STORAGE_POSTGRES_PASSWORD
+ password: XXXXXXXX
+
+# Configuration of the notification system.
+#
+# Notifications are sent to users when they require a password reset, a u2f
+# registration or a TOTP registration.
+# Use only an available configuration: filesystem, gmail
+notifier:
+ # For testing purpose, notifications can be sent in a file
+ ## filesystem:
+ ## filename: /tmp/authelia/notification.txt
+
+ # Use a SMTP server for sending notifications. Authelia uses PLAIN or LOGIN method to authenticate.
+ # [Security] By default Authelia will:
+ # - force all SMTP connections over TLS including unauthenticated connections
+ # - use the disable_require_tls boolean value to disable this requirement (only works for unauthenticated connections)
+ # - validate the SMTP server x509 certificate during the TLS handshake against the hosts trusted certificates
+ # - trusted_cert option:
+ # - this is a string value, that may specify the path of a PEM format cert, it is completely optional
+ # - if it is not set, a blank string, or an invalid path; will still trust the host machine/containers cert store
+ # - defaults to the host machine (or docker container's) trusted certificate chain for validation
+ # - use the trusted_cert string value to specify the path of a PEM format public cert to trust in addition to the hosts trusted certificates
+ # - use the disable_verify_cert boolean value to disable the validation (prefer the trusted_cert option as it's more secure)
+ smtp:
+ #username: test
+ # This secret can also be set using the env variables AUTHELIA_NOTIFIER_SMTP_PASSWORD
+ #password: password
+ #secure: false
+ host: smtp-server-hostname
+ port: 25
+ disable_require_tls: true
+ sender: abims@sb-roscoff.fr
+
+ # Sending an email using a Gmail account is as simple as the next section.
+ # You need to create an app password by following: https://support.google.com/accounts/answer/185833?hl=en
+ ## smtp:
+ ## username: myaccount@gmail.com
+ ## # This secret can also be set using the env variables AUTHELIA_NOTIFIER_SMTP_PASSWORD
+ ## password: yourapppassword
+ ## sender: admin@example.com
+ ## host: smtp.gmail.com
+ ## port: 587
--
GitLab