deploiement : support prod update ; also fix one bug ; huge refactoring

scripts/src/main.sh

@@ -5,7 +5,7 @@ ci_data_dir="/home/django/third-party/scripts/data"
 
 
 coop=$1
-instance=$2 #(prod not supported for now)
+instance=$2
 #move to the third argument
 shift
 shift
@@ -35,17 +35,26 @@ fi
 
 now=$( date +%Y%m%d_%H%M%S )
 
-source "${ci_dir}/update_code.sh"
+source "${ci_dir}/prepare_mail.sh"
+source "${ci_dir}/update_instance.sh"
+
+if [[ "${instance}" == "preprod" ]]; then
+  repo="third-party"
+  echo "Prepare mail" "${coop}" "${instance}" "${repo}" "${now}" "${send_mails}" "${ci_dir}" "${ci_data_dir}"
+  prepare_mail "${coop}" "${instance}" "${repo}" "${now}" "${send_mails}" "${ci_dir}" "${ci_data_dir}"|| exit 1
+
+  repo="Odoo"
+  echo "Prepare mail" "${coop}" "${instance}" "${repo}" "${now}" "${send_mails}" "${ci_dir}" "${ci_data_dir}"
+  prepare_mail "${coop}" "${instance}" "${repo}" "${now}" "${send_mails}" "${ci_dir}" "${ci_data_dir}" || exit 1
+fi
 
 repo="third-party"
-echo "START PROCESS" "${coop}" "${instance}" "${repo}" "${now}"
-update_code "${coop}" "${instance}" "${repo}" "${now}" "${send_mails}" || exit 1
-echo "DONE"
+echo "Update instance" "${coop}" "${instance}" "${repo}" "${now}" "${send_mails}" "${ci_dir}" "${ci_data_dir}"
+update_instance "${coop}" "${instance}" "${repo}" "${now}" "${send_mails}" "${ci_dir}" "${ci_data_dir}" || exit 1
 
 repo="Odoo"
-echo "START PROCESS" "${coop}" "${instance}" "${repo}" "${now}"
-update_code "${coop}" "${instance}" "${repo}" "${now}" "${send_mails}" || exit 1
-echo "DONE"
+echo "Update instance" "${coop}" "${instance}" "${repo}" "${now}" "${send_mails}" "${ci_dir}" "${ci_data_dir}"
+update_instance "${coop}" "${instance}" "${repo}" "${now}" "${send_mails}" "${ci_dir}" "${ci_data_dir}" || exit 1
 
 if [[ "${instance}" == "preprod" ]] && ${send_mails}; then
     python3 "${ci_dir}/send_mail.py" "${ci_data_dir}/ci_diff_third-party_${now}.txt" "${ci_data_dir}/ci_diff_Odoo_${now}.txt" "${coop}"

scripts/src/update_code.sh → scripts/src/prepare_mail.sh

-#This script prepares files to send an email and update code
-update_code() {
+prepare_mail() {
+  #Read arguments and set variables
   coop=$1
   instance=$2
   repo=$3
   now=$4
   send_mails=$5
-
-  ci_dir="/home/django/third-party/scripts/src"
-  ci_data_dir="/home/django/third-party/scripts/data"
-
-
-  ###EARLY CHECKS###
-
+  ci_dir=$6
+  ci_data_dir=$7
   if [[ ${repo} == "third-party" ]]; then
     user="django"
-    service="django"
   elif [[ ${repo} == "Odoo" ]]; then
     user="odoo"
-    service="odoo"
   else
     echo "Incorrect repo name : ${repo}"
     if ${send_mails}; then
@@ -29,6 +22,7 @@ update_code() {
   cd /home/${user}/${repo}
   su ${user} -c "git fetch --all"
 
+  #Check branch
   branch_found_str=$( su ${user} -c "git ls-remote origin ${coop}_prod" ) #seems that git ls-remote is not "failing" so we need to check output
   if [ -z "${branch_found_str}" ]; then
     #branch prod does not exist, cannot go on
@@ -81,115 +75,4 @@ update_code() {
     fi
   fi
 
-
-  ###UPDATE CODE (DO IT ANYWAY AS IT IS ONLY USED ON PREPROD FOR NOW)###
-
-  #If repo is third-party, make sure config file exists as we need it later
-  if [[ ${repo} == "third-party" ]]; then
-    if ! test -f "coops_configurations/config_${coop}.py"; then
-      echo "${coop} ${instance} ${repo}: coops_configurations/config_${coop}.py does not exists"
-      if ${send_mails}; then
-        python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : coops_configurations/config_${coop}.py does not exists"
-      fi
-      exit 1
-    fi
-  fi
-
-  /etc/init.d/${service} stop #TODO : on prod, make sure some changes are required before stopping / restarting service
-
-  #Stash mess, then checkout proper branch and pull
-  #Stash + checkout + pull seems a good way to get rid of mess without deleting it
-  su ${user} -c "git stash"
-  #maybe some user changed branch so we need to get back on dev_cooperatic
-  if ! su ${user} -c "git checkout dev_cooperatic"; then
-    #handling cases when git checkout branch fails
-    #We can not continue but we have stopped service already, let's restart it before exiting
-    /etc/init.d/${service} start
-    echo "${coop} ${instance} ${repo}: could not git checkout dev_cooperatic"
-    if ${send_mails}; then
-      python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : could not checkout dev_cooperatic"
-    fi
-    exit 1
-  fi
-
-  if [[ ${repo} == "Odoo" ]]; then
-    echo "git diff --name-only dev_cooperatic..origin/dev_cooperatic"
-    updated_filepaths="$( git diff --name-only dev_cooperatic..origin/dev_cooperatic )"
-    echo "${updated_filepaths}"
-  fi
-
-  if ! su ${user} -c "git pull"; then
-    #handling cases when git pull branch fails
-    #We can not continue but we have stopped service already, let's restart it before exiting
-    /etc/init.d/${service} start
-    echo "${coop} ${instance} ${repo}: could not git pull"
-    if ${send_mails}; then
-      python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : could not git pull"
-    fi
-    exit 1
-  fi
-
-  if [[ ${repo} == "third-party" ]]; then
-    #Replace config file by proper one only if required
-    if ! cmp "outils/config.py" "coops_configurations/config_${coop}.py"; then
-      #Config files are different, go for it
-      mv "outils/config.py" "${ci_data_dir}/ci_old_cfg_file_${now}.py"
-      cp "coops_configurations/config_${coop}.py" "outils/config.py"
-    else
-      echo "config files are identical"
-    fi
-    /etc/init.d/django start
-  else
-    #oop on updated_filenames, for each filename, look for the directory of __openerp__.py file
-    #if found, just store the parent directory
-    SAVEIFS=$IFS
-    IFS=$'\n'
-    if [[ ${updated_filepaths} == "" ]]; then
-      updated_filepaths_array=()
-    else
-      read -rd '' -a updated_filepaths_array <<<"${updated_filepaths}"
-    fi
-
-    IFS='/'
-    l=${#updated_filepaths_array[@]}
-
-    declare -A modules #modules to update
-    modules_str=""
-
-    for (( i=0; i<l; i++ ));
-    do
-      read -rd '' -a one_path_array <<<"${updated_filepaths_array[$i]}"
-      j=0
-      accu_path="${one_path_array[0]}"
-      pathlen=${#one_path_array[@]}
-      while [[ $j -lt ${pathlen} ]] && ! test -f "${accu_path}/__openerp__.py"; do
-        ((j++))
-        accu_path="${accu_path}/${one_path_array[j]}"
-      done
-      #Update key of modules if (we have found openerp.py (j<path) + module has not been added already),
-      #so that we do not add twice same module in modules_str
-      #and add module in modules_str
-      if [[ $j -lt ${pathlen} ]] && [[ ! -v modules["${one_path_array[$j]}"] ]]; then
-        modules["${one_path_array[$j]}"]=
-        if [[ ${modules_str} == "" ]]; then
-          modules_str="${one_path_array[$j]}"
-        else
-          modules_str="${modules_str},${one_path_array[$j]}"
-        fi
-      fi
-    done
-
-    IFS=$SAVEIFS
-
-    echo "${modules_str}"
-
-    restart_odoo_cmd="/home/odoo/Odoo/odoo/odoo.py -c /home/odoo/odoo-server.conf"
-    if [[ -n "${modules_str}" ]]; then
-      echo "restart odoo service with module update"
-      su odoo -c "${restart_odoo_cmd} -u${modules_str} --stop-after-init" #TODO : make sure this is working fine
-    fi
-  fi
-
-  echo "Starting ${service} service"
-  /etc/init.d/${service} start
 }
\ No newline at end of file

scripts/src/supported_coops.py

@@ -6,7 +6,7 @@ def is_coop_supported(coop):
 
 
 def is_instance_supported(instance):
-    return instance == "preprod" or instance == "test" or instance == "dev"
+    return instance == "preprod" or instance == "test" or instance == "dev" or instance == "prod"
 
 
 def get_mail(coop):

scripts/src/update_instance.sh

+update_instance() {
+  #Read arguments and set variables
+  coop=$1
+  instance=$2
+  repo=$3
+  now=$4
+  send_mails=$5
+  ci_dir=$6
+  ci_data_dir=$7
+  if [[ ${repo} == "third-party" ]]; then
+    user="django"
+    service="django"
+  elif [[ ${repo} == "Odoo" ]]; then
+    user="odoo"
+    service="odoo"
+  else
+    echo "Incorrect repo name : ${repo}"
+    if ${send_mails}; then
+      python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : incorrect repo name"
+    fi
+    exit 1
+  fi
+  if [[ "${instance}" == "prod" ]]; then
+    target="${coop}_prod"
+  else
+    target="dev_cooperatic"
+  fi
+
+  #Move to repository directory, fetch all and make sure coop_prod branch exists
+  cd /home/${user}/${repo}
+  su ${user} -c "git fetch --all"
+  branch_found_str=$( su ${user} -c "git ls-remote origin ${coop}_prod" ) #seems that git ls-remote is not "failing" so we need to check output
+  if [ -z "${branch_found_str}" ]; then
+    #branch prod does not exist, cannot go on
+    echo "${coop} ${instance} ${repo}: branch ${coop}_prod does not exist"
+    if ${send_mails}; then
+      python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : branch ${coop}_prod does not exist"
+    fi
+    exit 1
+  fi
+
+  #Make sure config file exists if repo is third-party
+  if [[ ${repo} == "third-party" ]]; then
+    if ! test -f "coops_configurations/config_${coop}.py"; then
+      echo "${coop} ${instance} ${repo}: coops_configurations/config_${coop}.py does not exists"
+      if ${send_mails}; then
+        python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : coops_configurations/config_${coop}.py does not exists"
+      fi
+      exit 1
+    fi
+  fi
+
+  #Save filepaths of files that are going to be updated if repo is odoo
+  if [[ ${repo} == "Odoo" ]]; then
+    echo "git diff --name-only origin/${target}.."
+    updated_filepaths="$( git diff --name-only origin/${target}.. )"
+    echo "${updated_filepaths}"
+  fi
+
+  #Update code if needed
+  new_commits="$( git log origin/${target}... )"
+  if [ -z "${new_commits}" ]; then
+    echo "${coop} ${instance} ${repo} : already up to date"
+  else
+    #Stop instance
+    /etc/init.d/${service} stop
+    #One could argue that git stash is risky on prod,
+    #but it's way easier to go that way to get rid of template file changes generated on django service restart
+    if ! su ${user} -c "git stash"; then
+      #We can not finish the update : restart the system
+      /etc/init.d/${service} start
+      echo "${coop} ${instance} ${repo}: could not git stash"
+      if ${send_mails}; then
+        python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : could not git stash"
+      fi
+      exit 1
+    fi
+    #Make script robust to any branch change : switch to target !
+    if ! su ${user} -c "git checkout ${target}"; then
+      #We can not finish the update : restart the system
+      #TODO : detect if we have stashed something on previous step to revert it (git stash pop)
+      /etc/init.d/${service} start
+      echo "${coop} ${instance} ${repo}: could not git checkout ${target}"
+      if ${send_mails}; then
+        python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : could not git checkout ${target}"
+      fi
+      exit 1
+    fi
+    #Pull
+    if ! su ${user} -c "git pull"; then
+      #We can not finish the update : restart the system
+      #TODO : detect if we have stashed something on previous step to revert it (git stash pop)
+      #TODO : undo previous git checkout if necessary
+      echo "${coop} ${instance} ${repo}: could not git pull"
+      if ${send_mails}; then
+        python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : could not git pull"
+      fi
+      exit 1
+    fi
+
+    #Additionnal update actions
+
+    #push tags to remember this prod update point (only on prod)
+    if [[ "${instance}" == "prod" ]]; then
+      #TODO : handle errors
+      su ${user} -c "git tag -a ${coop}_${now} -m autotag"
+      su ${user} -c "git push origin ${coop}_${now}"
+    fi
+
+    if [[ ${repo} == "third-party" ]]; then
+      #Replace config file by proper one only if required
+      if ! cmp "outils/config.py" "coops_configurations/config_${coop}.py"; then
+        #Config files are different, go for it
+        mv "outils/config.py" "${ci_data_dir}/ci_old_cfg_file_${now}.py"
+        cp "coops_configurations/config_${coop}.py" "outils/config.py"
+      else
+        echo "config files are identical"
+      fi
+    fi
+
+    if [[ ${repo} == "Odoo" ]]; then
+      #Loop on updated_filenames, for each filename, look for the directory of __openerp__.py file
+      #Ff found, just store the parent directory
+      SAVEIFS=$IFS
+      IFS=$'\n'
+      if [[ ${updated_filepaths} == "" ]]; then
+        updated_filepaths_array=()
+      else
+        read -rd '' -a updated_filepaths_array <<<"${updated_filepaths}"
+      fi
+
+      IFS='/'
+      l=${#updated_filepaths_array[@]}
+
+      declare -A modules #Modules to update
+      modules_str=""
+
+      for (( i=0; i<l; i++ ));
+      do
+        read -rd '' -a one_path_array <<<"${updated_filepaths_array[$i]}"
+        j=0
+        accu_path="${one_path_array[0]}"
+        pathlen=${#one_path_array[@]}
+        while [[ $j -lt ${pathlen} ]] && ! test -f "${accu_path}/__openerp__.py"; do
+          ((j++))
+          accu_path="${accu_path}/${one_path_array[j]}"
+        done
+        #Update key of modules if (we have found openerp.py (j<path) + module has not been added already),
+        #so that we do not add twice same module in modules_str
+        #and add module in modules_str
+        if [[ $j -lt ${pathlen} ]] && [[ ! -v modules["${one_path_array[$j]}"] ]]; then
+          modules["${one_path_array[$j]}"]=
+          if [[ ${modules_str} == "" ]]; then
+            modules_str="${one_path_array[$j]}"
+          else
+            modules_str="${modules_str},${one_path_array[$j]}"
+          fi
+        fi
+      done
+
+      IFS=$SAVEIFS
+
+      echo "${modules_str}"
+
+      restart_odoo_cmd="/home/odoo/Odoo/odoo/odoo.py -c /home/odoo/odoo-server.conf"
+      if [[ -n "${modules_str}" ]]; then
+        echo "restart odoo service with module update"
+        su odoo -c "${restart_odoo_cmd} -u${modules_str} --stop-after-init"
+      fi
+    fi
+
+    #Restart service
+    echo "Starting ${service} service"
+    /etc/init.d/${service} start
+  fi
+}
\ No newline at end of file