various documentation fixes

Author: bgruening

.github/workflows/single.sh

@@ -164,6 +164,17 @@ if [[ "$SKIP_SFTP" != "true" ]]; then
     sshpass -p $GALAXY_USER_PASSWD sftp -v -P 8022 -o User=$GALAXY_USER -o "StrictHostKeyChecking no" localhost <<< $'put time.txt'
 fi
 
+# Test FTP Server from within the container (avoids host NAT/passive issues)
+docker_exec python - <<'PY'
+import ftplib
+
+ftp = ftplib.FTP()
+ftp.connect("localhost", 21, timeout=30)
+ftp.login("admin@example.org", "password")
+ftp.retrlines("LIST")
+ftp.quit()
+PY
+
 # Test CVMFS
 docker_exec bash -c "service autofs start"
 docker_exec bash -c "cvmfs_config chksetup"

README.md

@@ -14,7 +14,7 @@ One of the main goals is to make access to entire tool suites as easy as possibl
 this includes the setup of a public available web-service that needs to be maintained, or that the Tool-user needs to either setup a Galaxy Server by its own or to have Admin access to a local Galaxy server.
 With docker, tool developers can create their own Image with all dependencies and the user only needs to run it within docker.
 
-The Image is based on [Ubuntu 22.04 LTS](http://releases.ubuntu.com/22.04/) and all recommended Galaxy requirements are installed. The following chart should illustrate the [Docker](http://www.docker.io) image hierarchy we have build to make is as easy as possible to build on different layers of our stack and create many exciting Galaxy flavors.
+The Image is based on [Ubuntu 24.04 LTS](http://releases.ubuntu.com/24.04/) and all recommended Galaxy requirements are installed. The following chart should illustrate the [Docker](http://www.docker.io) image hierarchy we have build to make is as easy as possible to build on different layers of our stack and create many exciting Galaxy flavors.
 
 ![Docker hierarchy](https://raw.githubusercontent.com/bgruening/docker-galaxy-stable/master/chart.png)
 
@@ -115,6 +115,10 @@ With the additional `-v /home/user/galaxy_storage/:/export/` parameter, Docker w
 
 This enables you to have different export folders for different sessions - means real separation of your different projects.
 
+To detect when the Galaxy distribution in the image changes, the container writes a marker at
+`/export/.galaxy_export_marker`. You can override the marker value with `GALAXY_EXPORT_MARKER` if you
+need deterministic export refresh behavior.
+
 You can also collect and store `/export/` data of Galaxy instances in a dedicated docker [Data  volume Container](https://docs.docker.com/engine/userguide/dockervolumes/) created by:
 
 ```sh
@@ -517,7 +521,16 @@ This is achieved by connecting to Galaxy's CernVM filesystem (CVMFS) at `cvmfs-c
 The CVMFS capability doesn't add to the size of the Docker image, but when running, CVMFS maintains
 a cache to keep the most recently used data on the local disk.
 
-*Note*: for CVMFS directories to be mounted-on-demand with `autofs`, you must launch Docker as `--privileged`
+*Note*: for CVMFS directories to be mounted-on-demand with `autofs`, you must launch Docker as `--privileged`.
+If privileged mode is not an option, use the optional CVMFS sidecar in `galaxy/docker-compose.yaml`:
+
+```sh
+cd galaxy
+CVMFS_MOUNT_DIR=/cvmfs EXPORT_DIR=./export docker compose --profile cvmfs up
+```
+
+This starts a dedicated CVMFS container that mounts the repositories and shares `/cvmfs` with the Galaxy
+container. The CVMFS cache is persisted in `${EXPORT_DIR}/cvmfs-cache`.
 
 
 ## Personalize your Galaxy <a name="Personalize-your-Galaxy" /> [[toc]](#toc)
@@ -748,6 +761,7 @@ When you execute the tool again, Galaxy will pull the image from Biocontainers (
 | `LOAD_GALAXY_CONDITIONAL_DEPENDENCIES` | Installing optional dependencies into the Galaxy virtual environment |
 | `LOAD_PYTHON_DEV_DEPENDENCIES` | Installation of Galaxy's dev dependencies. Needs `LOAD_GALAXY_CONDITIONAL_DEPENDENCIES` as well |
 | `GALAXY_AUTO_UPDATE_DB` | Run the Galaxy database migration script during startup |
+| `GALAXY_EXPORT_MARKER` | Override the export marker used to refresh `/export/galaxy`. |
 
 
 # HTTPS Support <a name="HTTPS-Support"/> [[toc]](#toc)

cvmfs/Dockerfile

@@ -25,4 +25,4 @@ COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
 RUN chmod 0755 /usr/local/bin/docker-entrypoint.sh
 
 ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
-CMD ["tail", "-f", "/dev/null"]
+CMD ["bash", "-lc", "tail -F /var/log/autofs.log /var/log/cvmfs.log 2>/dev/null"]

cvmfs/docker-entrypoint.sh

@@ -6,6 +6,7 @@ repos="${repos//,/ }"
 
 mkdir -p /cvmfs
 mkdir -p "${CVMFS_CACHE_BASE:-/var/lib/cvmfs}"
+touch /var/log/autofs.log /var/log/cvmfs.log
 
 if [[ ! -f "${CVMFS_CACHE_BASE:-/var/lib/cvmfs}/.configured" ]]; then
     ansible-playbook /ansible/playbook.yml

galaxy/Dockerfile

@@ -174,6 +174,8 @@ RUN apt-get -qq update \
     # Install ansible roles
     && ansible-galaxy install -r /ansible/requirements.yml -p /ansible/roles \
     && npm install -g @galaxyproject/gx-it-proxy@latest \
+    && apt-get purge -y npm \
+    && apt-get autoremove -y \
     && groupadd -r $GALAXY_USER -g $GALAXY_GID \
     && useradd -u $GALAXY_UID -r -g $GALAXY_USER -d $GALAXY_HOME -m -c "Galaxy user" --shell /bin/bash $GALAXY_USER \
     # Create the postgres user before apt-get does (with the configured UID/GID) to facilitate sharing $EXPORT_DIR/postgresql with non-Linux hosts
@@ -292,6 +294,7 @@ RUN mkdir -p $GALAXY_CONFIG_TUS_UPLOAD_STORE \
     && curl -o $GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH \
     -L https://raw.githubusercontent.com/galaxyproject/usegalaxy-playbook/8adb1f82c94fe95b09df2a2816440ce2420b7d39/env/main/files/galaxy/config/tool_data_table_conf.xml \
     && chown $GALAXY_USER:$GALAXY_USER $GALAXY_CONFIG_TOOL_DATA_TABLE_CONFIG_PATH \
+    # Ensure Galaxy uses the jemalloc we built (gridengine compatibility: #10425).
     && mv /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 /usr/lib/x86_64-linux-gnu/libjemalloc.so.2.orig \
     && ln -s /usr/local/lib/libjemalloc.so.2 /usr/lib/x86_64-linux-gnu/libjemalloc.so.2
 

galaxy/ansible/nginx.yml

@@ -2,6 +2,8 @@
   connection: local
   remote_user: root
   vars:
+    # Default container config: avoid DH param generation and OCSP stapling
+    # errors in ephemeral/self-signed setups. Override in production.
     nginx_conf_ssl_protocols:
       - TLSv1.2
       - TLSv1.3

galaxy/ansible/rabbitmq.yml

@@ -4,6 +4,7 @@
   vars:
     rabbitmq_keyring_path: /usr/share/keyrings/com.rabbitmq.team.gpg
     rabbitmq_repo_list_path: /etc/apt/sources.list.d/rabbitmq.list
+    rabbitmq_version: 4.2.2-1
     rabbitmq_erlang_packages:
       - erlang-base
       - erlang-asn1
@@ -58,7 +59,7 @@
 
     - name: Install RabbitMQ server
       apt:
-        name: rabbitmq-server
+        name: "rabbitmq-server={{ rabbitmq_version }}"
         state: present
         update_cache: true
 

galaxy/ansible/requirements.yml

@@ -5,7 +5,6 @@ roles:
   - name: geerlingguy.docker
     version: 7.9.0
   - name: usegalaxy_eu.flower
-    src: https://github.com/usegalaxy-eu/flower-ansible-role
     version: 2.1.1
   - name: grycap.htcondor
     src: https://github.com/usegalaxy-eu/ansible-htcondor-grycap
@@ -27,7 +26,6 @@ roles:
   - name: galaxyproject.nginx
     version: 1.0.0
   - name: usegalaxy_eu.certbot
-    src: https://github.com/usegalaxy-eu/ansible-certbot
     version: 0.1.13
   - name: galaxyproject.self_signed_certs
     version: 0.0.4

galaxy/ansible/templates/export_user_files.py.j2

@@ -12,7 +12,7 @@ PG_VERSION = os.environ.get('PG_VERSION', '15')
 GALAXY_UID = int(os.environ['GALAXY_UID'])
 GALAXY_GID = int(os.environ['GALAXY_GID'])
 GALAXY_ROOT_DIR = os.environ.get('GALAXY_ROOT_DIR', '/galaxy/')
-GALAXY_IMAGE_MARKER_PATH = '/export/.galaxy_image_marker'
+GALAXY_EXPORT_MARKER_PATH = '/export/.galaxy_export_marker'
 
 if len( sys.argv ) == 2:
     PG_DATA_DIR_DEFAULT = sys.argv[1]
@@ -73,14 +73,15 @@ def _ignore_static(dir, *patterns):
     return __ignore_static
 
 def _read_image_marker():
-    marker = os.environ.get('GALAXY_IMAGE_MARKER')
+    marker = os.environ.get('GALAXY_EXPORT_MARKER')
     if marker:
         return marker.strip()
     version_py = os.path.join(GALAXY_ROOT_DIR, 'lib', 'galaxy', 'version.py')
     if os.path.exists(version_py):
         try:
             with open(version_py, 'r', encoding='utf-8', errors='ignore') as handle:
                 text = handle.read()
+            # Extract __version__ without importing Galaxy modules during startup.
             match = re.search(r'__version__\\s*=\\s*[\\\'"]([^\\\'"]+)[\\\'"]', text)
             if match:
                 return f"version:{match.group(1)}"
@@ -104,26 +105,27 @@ def _should_copy_distribution(marker):
     if not marker:
         return True
     try:
-        with open(GALAXY_IMAGE_MARKER_PATH, 'r', encoding='utf-8') as handle:
+        # Explicit UTF-8 decoding avoids locale-dependent behavior for marker files.
+        with open(GALAXY_EXPORT_MARKER_PATH, 'r', encoding='utf-8') as handle:
             return handle.read().strip() != marker
     except OSError:
         return True
 
 def _write_marker(marker):
     if not marker:
         return
-    tmp_path = f"{GALAXY_IMAGE_MARKER_PATH}.tmp"
+    tmp_path = f"{GALAXY_EXPORT_MARKER_PATH}.tmp"
     with open(tmp_path, 'w', encoding='utf-8') as handle:
         handle.write(marker + '\n')
-    os.replace(tmp_path, GALAXY_IMAGE_MARKER_PATH)
+    os.replace(tmp_path, GALAXY_EXPORT_MARKER_PATH)
 
 if __name__ == "__main__":
     """
         If the '/export/' folder exist, meaning docker was started with '-v /home/foo/bar:/export',
-        we will link every file that needs to persist to the host system. Addionaly a file (/.galaxy_save) is
-        created that indicates all linking is already done.
-        If the user re-starts (with docker start) the container the file /.galaxy_save is found and the linking
-        is aborted.
+        we will link every file that needs to persist to the host system. A marker file at
+        /export/.galaxy_export_marker is written to indicate the export contents match the image version.
+        If the user re-starts (with docker start) the container and the marker matches, the linking
+        is skipped.
     """
     marker = _read_image_marker()
     if _should_copy_distribution(marker):

galaxy/ansible/templates/supervisor.conf.j2

@@ -32,7 +32,8 @@ stdout_logfile  = /var/log/autofs.log
 [program:munge]
 user=root
 # In VMs the chown seems to be needed, in containers the mkdir.
-command=/bin/bash -c "mkdir -p /var/run/munge && chown -R root:root /var/run/munge && /usr/sbin/munged -f -F --num-threads=10"
+# Keep munge threads modest by default; increase via munge_num_threads if needed.
+command=/bin/bash -c "mkdir -p /var/run/munge && chown -R root:root /var/run/munge && /usr/sbin/munged -f -F --num-threads={{ munge_num_threads | default(2) }}"
 redirect_stderr = true
 priority        = 100
 stopasgroup     = true