From 8687fdab47ad6e59bca7aba21163c45386a40c3d Mon Sep 17 00:00:00 2001 From: Andrea Dell'Amico Date: Thu, 14 Sep 2017 13:30:25 +0200 Subject: [PATCH 1/4] Add new tags to the task that creates the downloads directory so that we can use it without running all the smartgears role. --- smartgears/smartgears/tasks/smartgears-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smartgears/smartgears/tasks/smartgears-app.yml b/smartgears/smartgears/tasks/smartgears-app.yml index c8f6f6d0..0460b31c 100644 --- a/smartgears/smartgears/tasks/smartgears-app.yml +++ b/smartgears/smartgears/tasks/smartgears-app.yml @@ -1,7 +1,7 @@ --- - name: Create a directory where to store all the smartgears related downloads to avoid cluttering the home directory file: dest={{ smartgears_downloads_dir }} state=directory - tags: [ 'smartgears', 'tomcat' ] + tags: [ 'smartgears', 'tomcat', 'smartgears_download_dir', 'smartgears_downloads_dir' ] - name: Remove the smartgears distribution archive to force un upgrade file: dest={{ item }} state=absent From 22632aa6349fd8b08669b0e62952a3334df00876 Mon Sep 17 00:00:00 2001 From: Andrea Dell'Amico Date: Thu, 14 Sep 2017 13:41:55 +0200 Subject: [PATCH 2/4] dataminer: Fix the algorithms installer script, in some situations it created duplicated lines in the file that lists the already installed algorithms. dataminer: Download the distribution files into /home/gcube/downloads --- smartgears/dataminer_app/tasks/dataminer-app.yml | 10 ++++++---- .../dataminer_app/templates/algorithms-updater.j2 | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/smartgears/dataminer_app/tasks/dataminer-app.yml b/smartgears/dataminer_app/tasks/dataminer-app.yml index defd1bfc..4ad06b38 100644 --- a/smartgears/dataminer_app/tasks/dataminer-app.yml +++ b/smartgears/dataminer_app/tasks/dataminer-app.yml @@ -1,4 +1,5 @@ --- +# This is only needed to force an upgrade - block: - name: Remove the old dataminer installation file: path={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }} state=absent @@ -10,6 +11,7 @@ - dataminer_app_upgrade tags: [ 'tomcat', 'dataminer', 'wps' ] +# This is only needed to force an upgrade - block: - name: Remove the old dataminer algorithm installer file: path={{ smartgears_user_home }}/algorithmInstaller state=absent @@ -23,7 +25,7 @@ - block: - name: Get and unpack the dataminer application - maven_artifact: artifact_id={{ dataminer_wps_name }} version={{ dataminer_wps_version | default(omit) }} group_id={{ dataminer_wps_group_id }} extension={{ dataminer_wps_extension | default('war') }} repository_url={{ smartgears_global_base_url }} dest={{ smartgears_user_home }}/{{ dataminer_wps_file }} + maven_artifact: artifact_id={{ dataminer_wps_name }} version={{ dataminer_wps_version | default(omit) }} group_id={{ dataminer_wps_group_id }} extension={{ dataminer_wps_extension | default('war') }} repository_url={{ smartgears_global_base_url }} dest={{ smartgears_downloads_dir }}/{{ dataminer_wps_file }} register: dataminer_app_downloaded - name: Remove the old dataminer installation @@ -34,7 +36,7 @@ file: path={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }} state=directory owner={{ smartgears_user }} group={{ smartgears_user }} - name: Unarchive the dataminer_wps war - unarchive: copy=no src={{ smartgears_user_home }}/{{ dataminer_wps_file }} dest={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }} creates='{{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }}/WEB-INF/lib' + unarchive: copy=no src={{ smartgears_downloads_dir }}/{{ dataminer_wps_file }} dest={{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }} creates='{{ smartgears_instance_path }}/webapps/{{ dataminer_wps_name }}/WEB-INF/lib' notify: Restart smartgears - name: Install the dataminer web.xml @@ -45,7 +47,7 @@ file: path={{ smartgears_user_home }}/wps_algorithms state=directory - name: Get and unpack the algorithms installer - maven_artifact: artifact_id={{ dataminer_algorithms_name }} version={{ dataminer_algorithms_version | default(omit) }} group_id={{ dataminer_algorithms_group_id }} extension={{ dataminer_algorithms_extension | default('war') }} repository_url={{ dataminer_algorithms_base_url }} dest={{ smartgears_user_home }}/{{ dataminer_algorithms_file }} + maven_artifact: artifact_id={{ dataminer_algorithms_name }} version={{ dataminer_algorithms_version | default(omit) }} group_id={{ dataminer_algorithms_group_id }} extension={{ dataminer_algorithms_extension | default('war') }} repository_url={{ dataminer_algorithms_base_url }} dest={{ smartgears_downloads_dir }}/{{ dataminer_algorithms_file }} register: algorithms_installer_downloaded tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ] @@ -55,7 +57,7 @@ tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ] - name: Unarchive the algorithms installer - unarchive: copy=no src={{ smartgears_user_home }}/{{ dataminer_algorithms_file }} dest={{ smartgears_user_home }} creates='{{ smartgears_user_home }}/algorithmInstaller/addAlgorithm.sh' + unarchive: copy=no src={{ smartgears_downloads_dir }}/{{ dataminer_algorithms_file }} dest={{ smartgears_user_home }} creates='{{ smartgears_user_home }}/algorithmInstaller/addAlgorithm.sh' tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_installer' ] - name: Download the WPS algorithms from subversion diff --git a/smartgears/dataminer_app/templates/algorithms-updater.j2 b/smartgears/dataminer_app/templates/algorithms-updater.j2 index 1323199e..879bae02 100644 --- a/smartgears/dataminer_app/templates/algorithms-updater.j2 +++ b/smartgears/dataminer_app/templates/algorithms-updater.j2 @@ -12,6 +12,7 @@ ALGORITHMS_TEMP_SCRIPT=$OUT_DIR/add_algorithms LOG_DIR={{ smartgears_user_home }}/wps_algorithms_install_log LOG_FILE=${LOG_DIR}/algorithms_updater.log ALGORITHMS_INSTALLED_FILE=${LOG_DIR}/already_installed_algorithms.txt +ALGORITHMS_INSTALLED_SORTED_FILE=${LOG_DIR}/already_installed_algorithms_sorted.txt LOCK_FILE=${LOG_DIR}/.algorithms_updater.lock ALGO_DIR={{ dataminer_wps_algorithms_dest }} RUNNING_JOB= @@ -60,6 +61,13 @@ function update_svn_repo() { svn update > $LOG_FILE 2>&1 } +function sort_installed_algo_file() { + if [ -f $ALGORITHMS_INSTALLED_FILE ] ; then + sort -u $ALGORITHMS_INSTALLED_FILE > $ALGORITHMS_INSTALLED_SORTED_FILE + mv $ALGORITHMS_INSTALLED_SORTED_FILE $ALGORITHMS_INSTALLED_FILE + fi +} + function algorithms_updater() { logger 'algorithms-updater: scan the algorithms list and build the algorithms script. Reference infra is {{ dataminer_infra_reference }}' echo "#!/bin/bash" > $ALGORITHMS_TEMP_SCRIPT.head @@ -80,6 +88,9 @@ function algorithms_updater() { ALGO_BODY=$( echo "$algo" | awk -F \| '{ print $2 }' ) ALGO_DATE=$( echo "$algo" | awk -F \| '{ print $3 }' ) ALGO_TIMESTAMP=$( date +%s -d "$ALGO_DATE" ) + if [ $? -ne 0 ] ; then + echo "the date for algorithm $ALGO_NAME is wrong" + fi ALGO_LINE=$( egrep ^"$ALGO_NAME" $ALGORITHMS_INSTALLED_FILE ) ALGO_PRESENT=$? if [ $ALGO_PRESENT -ne 0 ] ; then @@ -106,6 +117,9 @@ function algorithms_updater() { ALGO_LOCAL_TIMESTAMP=0 ALGO_LOCAL_DATE=$( egrep ^"$ALGO_NAME" $ALGORITHMS_INSTALLED_FILE | awk -F \| '{ print $2 }' | grep -v \' ) ALGO_LOCAL_TIMESTAMP=$( date +%s -d "$ALGO_LOCAL_DATE" ) + if [ $? -ne 0 ] ; then + echo "the locally saved date for algorithm $ALGO_NAME is wrong" + fi if [ $ALGO_LOCAL_TIMESTAMP -ne $ALGO_TIMESTAMP ] ; then echo "logger 'algorithms-updater: running the add command of algorithm $ALGO_NAME'" >> $ALGORITHMS_TEMP_SCRIPT.body_ echo "logger 'algorithms-updater: the add command string is $ALGO_BODY'" >> $ALGORITHMS_TEMP_SCRIPT.body_ @@ -155,5 +169,6 @@ function algorithms_updater() { create_log_dir check_lock_file update_svn_repo +sort_installed_algo_file algorithms_updater cleanup 0 From 11f5be1a837e765e07ce46e8243eb8fe09b99749 Mon Sep 17 00:00:00 2001 From: Andrea Dell'Amico Date: Thu, 14 Sep 2017 14:18:22 +0200 Subject: [PATCH 3/4] dataminer: add variables that permit to change the algorithms installer run frequency. Default: 1 hour. dataminer proto: run the algorithm installer every minute. --- smartgears/dataminer_app/defaults/main.yml | 2 ++ smartgears/dataminer_app/tasks/dataminer-app.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/smartgears/dataminer_app/defaults/main.yml b/smartgears/dataminer_app/defaults/main.yml index 62e27c35..72d6d0b4 100644 --- a/smartgears/dataminer_app/defaults/main.yml +++ b/smartgears/dataminer_app/defaults/main.yml @@ -26,6 +26,8 @@ dataminer_algorithms_base_url: 'http://maven.research-infrastructures.eu/nexus/c # Algorithms updater dataminer_wps_algorithms_svn: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/algorithms/ dataminer_wps_algorithms_dest: '{{ smartgears_user_home }}/wps_algorithms/algorithms' +dataminer_algorithms_updater_minute_frequency: "0" +dataminer_algorithms_updater_hour_frequency: "*" dataminer_data_file_from_thredds: 'http://thredds.d4science.org/thredds/fileServer/public/netcdf/gebco_08_OCEANS_CLIMATOLOGY_METEOROLOGY_ATMOSPHERE_.nc' diff --git a/smartgears/dataminer_app/tasks/dataminer-app.yml b/smartgears/dataminer_app/tasks/dataminer-app.yml index 4ad06b38..b37e8ed0 100644 --- a/smartgears/dataminer_app/tasks/dataminer-app.yml +++ b/smartgears/dataminer_app/tasks/dataminer-app.yml @@ -74,7 +74,7 @@ tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_script' ] - name: Cron job that updates the algorithms repository and adds the missing algorithms configurations - cron: name="SVN update the algorithms repository" minute="*/1" job="/usr/local/bin/algorithms-updater > {{ smartgears_user_home }}/wps_algorithms_install_log/algorithms_updater_cron.log 2>&1" user='{{ smartgears_user }}' state=present + cron: name="SVN update the algorithms repository" minute="{{ dataminer_algorithms_updater_minute_frequency }}" hour="{{ dataminer_algorithms_updater_hour_frequency }}" job="/usr/local/bin/algorithms-updater > {{ smartgears_user_home }}/wps_algorithms_install_log/algorithms_updater_cron.log 2>&1" user='{{ smartgears_user }}' state=present tags: [ 'tomcat', 'dataminer', 'wps', 'dataminer_algorithms', 'dataminer_algorithms_cron' ] - name: Create a directory where to install the gebco_08.nc data file From 7378e765f19a9b4d250425ba34e7daaff1ea01cb Mon Sep 17 00:00:00 2001 From: Andrea Dell'Amico Date: Thu, 14 Sep 2017 17:05:14 +0200 Subject: [PATCH 4/4] Fix the social-networking-library-ws role. See https://support.d4science.org/issues/9688 --- .../defaults/main.yml | 4 +--- .../social-networking-library-ws/tasks/main.yml | 17 ++++++++--------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/smartgears/social-networking-library-ws/defaults/main.yml b/smartgears/social-networking-library-ws/defaults/main.yml index ead6da66..4a5c0c81 100644 --- a/smartgears/social-networking-library-ws/defaults/main.yml +++ b/smartgears/social-networking-library-ws/defaults/main.yml @@ -1,10 +1,8 @@ --- social_networking_library_ws_install: False -social_networking_library_ws_version: 2.1.0-4.6.1-152935 -#social_networking_library_ws_version: latest +social_networking_library_ws_version: latest social_networking_library_ws_name: social-networking-library-ws social_networking_library_ws_group_id: org.gcube.portal social_networking_library_ws_extension: war social_networking_library_ws_file: '{{ social_networking_library_ws_name }}-{{ social_networking_library_ws_version }}.{{ social_networking_library_ws_extension }}' -social_networking_library_ws_repository_url: 'http://maven.research-infrastructures.eu/nexus/content/repositories/{{ gcube_repository }}' diff --git a/smartgears/social-networking-library-ws/tasks/main.yml b/smartgears/social-networking-library-ws/tasks/main.yml index dac5fab3..6877b833 100644 --- a/smartgears/social-networking-library-ws/tasks/main.yml +++ b/smartgears/social-networking-library-ws/tasks/main.yml @@ -1,23 +1,22 @@ --- - block: - name: Remove the old social-networking-library-ws files - file: path={{ smartgears_instance_path }}/webapps/{{ social_networking_library_ws_name }} state=absent + file: path={{ item }} state=absent + with_items: + - '{{ smartgears_instance_path }}/webapps/{{ social_networking_library_ws_name }}' + - '{{ smartgears_instance_path }}/webapps/{{ social_networking_library_ws_name }}.{{ social_networking_library_ws_extension }}' become: True become_user: '{{ d4science_user }}' - when: ( not social_networking_library_ws_install ) or ( smartgears_upgrade ) + when: not social_networking_library_ws_install tags: [ 'smartgears', 'social_networking_library_ws', 'tomcat' ] - block: - # NOTE: Install as the smartgears user so we do not mess with the permissions - name: Get the social-networking-library-ws war - maven_artifact: artifact_id={{ social_networking_library_ws_name }} version={{ social_networking_library_ws_version | default(omit) }} group_id={{ social_networking_library_ws_group_id }} extension={{ social_networking_library_ws_extension | default('war') }} repository_url={{ social_networking_library_ws_repository_url }} dest={{ smartgears_user_home }}/{{ social_networking_library_ws_file }} + maven_artifact: artifact_id={{ social_networking_library_ws_name }} version={{ social_networking_library_ws_version | default(omit) }} group_id={{ social_networking_library_ws_group_id }} extension={{ social_networking_library_ws_extension | default('war') }} repository_url={{ smartgears_global_base_url }} dest={{ smartgears_downloads_dir }}/{{ social_networking_library_ws_file }} - - name: Create the data trasfer working directory - file: path={{ smartgears_instance_path }}/webapps/{{ social_networking_library_ws_name }} state=directory owner={{ smartgears_user }} group={{ smartgears_user }} - - - name: Unarchive the social_networking_library_ws war - unarchive: copy=no src={{ smartgears_user_home }}/{{ social_networking_library_ws_file }} dest={{ smartgears_instance_path }}/webapps/{{ social_networking_library_ws_name }} creates={{ smartgears_instance_path }}/webapps/{{ social_networking_library_ws_name }}/WEB-INF/lib + - name: Copy the social_networking_library_ws war into the tomcat webapps dir + copy: src={{ smartgears_downloads_dir }}/{{ social_networking_library_ws_file }} dest={{ smartgears_instance_path }}/webapps/{{ social_networking_library_ws_name }}.{{ social_networking_library_ws_extension }} remote_src=yes force=yes notify: Restart smartgears become: True