Merge branch 'CI_for_mutualisation' into 'dev_cooperatic'

CI scripts for mutualisation : v0 (only for preprods)

See merge request !251

.gitignore

@@ -20,3 +20,4 @@ shop/errors.log
 members/settings.json
 .DS_Store
 exec.*.log
+scripts/src/ci_secrets.py
\ No newline at end of file

scripts/src/ci_secrets.example.py

+mail_server_password = "change me !"
+
+coops_mails = {
+    'lacagette': 'change me !',
+    'supercafoutch': 'change me !',
+    'graoucoop': 'change me !'
+}
+
+coops_preprod_urls = {
+    'lacagette': 'https://preprod.odoo.lacagette-coop.fr',
+    'supercafoutch': 'https://preprod.odoo.supercafoutch.cooperatic.fr/',
+    'graoucoop': 'https://preprod.odoo.graoucoop.cooperatic.fr'
+}
\ No newline at end of file

scripts/src/main.sh

+#!/bin/bash
+
+# $1 : project name
+# $2 : prod or preprod (only preprod is supported for now)
+
+ci_dir="/home/django/third-party/scripts/src"
+ci_data_dir="/home/django/third-party/scripts/data"
+
+if ! python3 "${ci_dir}/supported_coops.py" "${1}" "${2}"; then
+  echo "arguments : ${1} ${2} not supported"
+  python3 "${ci_dir}/send_error_mail.py" "arguments : ${1} ${2} not supported"
+  exit 1
+fi
+
+now=$( date +%Y%m%d_%H%M%S )
+
+source "${ci_dir}/update_code.sh"
+
+repo="third-party"
+echo "START PROCESS" "${1}" "${2}" "${repo}" "${now}"
+update_code "${1}" "${2}" "${repo}" "${now}" || exit 1
+echo "DONE"
+
+repo="Odoo"
+echo "START PROCESS" "${1}" "${2}" "${repo}" "${now}"
+update_code "${1}" "${2}" "${repo}" "${now}" || exit 1
+echo "DONE"
+
+python3 "${ci_dir}/send_mail.py" "${ci_data_dir}/ci_diff_third-party_${now}.txt" "${ci_data_dir}/ci_diff_Odoo_${now}.txt" "${1}"
\ No newline at end of file

scripts/src/old-to-new-env-steps.txt

+//Ces notes ont été prises lors de la conversion de l\'architecture cagette préprod vers la nouvelle architecture.
+//Elles peuvent servir à faire de même sur la prod !
+
+//USERS, HOME DIRS, FILES, RIGHTS
+useradd -m odoo
+usermod -s /bin/bash odoo
+useradd -m django
+usermod -s /bin/bash django
+cd /home/lacagette
+mv Odoo odoo-server.conf odoo-server.log odoo-server.log.1 odoo-server.log.2 scripts .local /home/odoo/.
+//un nouveau odoo-server.log a été créé automatiquement, il faudra gérer ça
+mv -f third-party cronlogs .virtualenvs /home/django/.
+mkdir /home/odoo/cronlogs
+cp -r /home/odoo/scripts /home/django/scripts
+chown -R odoo:odoo /home/odoo
+chown -R django:django /home/django
+chown -R django:django /shared_dir/django_static
+
+//PYTHON VENV
+cd /home/django/.virtualenvs
+rm -rf third-party .gitignore bin lib pyvenv.cfg
+source /usr/local/bin/virtualenvwrapper.sh
+mkvirtualenv -p python3.8 -r /home/django/third-party/requirements.txt third-party
+
+//PSQL
+//key to understand : "A user is a role with the ability to log in (the role has the LOGIN attribute)."
+su postgres
+psql
+create role odoo login;
+grant lacagette to odoo;
+alter database lacagette owner to odoo;
+select pg_terminate_backend(pid) from pg_stat_activity where pid <> pg_backend_pid() and datname = 'lacagette';
+alter database lacagette rename to odoo;
+exit
+exit
+
+//CONFIGS
+sed -i -e 's/\/lacagette\//\/odoo\//g' /home/odoo/odoo-server.conf
+sed -i -e 's/db_name = lacagette/db_name = odoo/g' /home/odoo/odoo-server.conf
+sed -i -e 's/db_user = lacagette/db_user = odoo/g' /home/odoo/odoo-server.conf
+sed -i -e 's/dbfilter = lacagette/dbfilter = odoo/g' /home/odoo/odoo-server.conf
+sed -i -e "s/'lacagette'/'odoo'/g" /home/django/third-party/outils/settings_secret.py
+sed -i -e 's/lacagette/django/g' /etc/scripts/django.sh
+sed -i -e 's/lacagette/django/g' /etc/init.d/django
+sed -i -e 's/lacagette/odoo/g' /etc/init.d/odoo
+
+//CRONS
+crontab -l -u lacagette | grep -v "sync_files.sh" | crontab -u django -
+crontab -l -u lacagette | grep "sync_files.sh" | crontab -u odoo -
+crontab -r -u lacagette
+
+//FILESTORE ISSUE
+su postgres
+psql
+\c odoo
+delete from ir_attachment where mimetype in ('text/css','application/javascript');
+//puis deux restarts nécessaires... de odoo
+
+restart django.

scripts/src/send_error_mail.py

+import smtplib
+import sys
+from email.message import EmailMessage
+import ci_secrets
+
+
+def run(body):
+    msg = EmailMessage()
+    msg['Subject'] = "CoopdevCI : script error"
+    msg['From'] = "brevo@ksuite-coopdev.fr"
+    msg['To'] = "yvon.kerdoncuff@gmail.com"
+    msg.set_content(body)
+
+    s = smtplib.SMTP('smtp-relay.sendinblue.com', port=587)
+    s.login("brevo@ksuite-coopdev.fr", ci_secrets.mail_server_password)
+    s.send_message(msg)
+    s.quit()
+
+
+if __name__ == '__main__':
+    arg = ""
+    if len(sys.argv) > 1:
+        arg = sys.argv[1]
+    run(arg)

scripts/src/send_mail.py

+import smtplib
+import sys
+from email.message import EmailMessage
+import ci_secrets
+import os
+
+
+def run(tp_file,od_file,coop):
+    msg = EmailMessage()
+    msg['Subject'] = "Coopdev : évolutions à tester"
+    msg['From'] = "brevo@ksuite-coopdev.fr"
+    msg['To'] = ci_secrets.coops_mails[coop]
+    msg['CC'] = "assistance-redmine@coopdev.fr"
+    msg['Reply-to'] = "assistance-redmine@coopdev.fr"
+
+    tp_file_exists = os.path.exists(tp_file)
+    od_file_exists = os.path.exists(od_file)
+
+    if not tp_file_exists and not od_file_exists:
+        print("MAIL : rien à envoyer")
+        return
+    else:
+        body = "Cher client, chère cliente,\n"
+        body = body + "Les évolutions suivantes ont été mises en ligne sur " + ci_secrets.coops_preprod_urls[coop] + ".\n"
+        body = body + "Merci de tester leur fonctionnement et nous faire un retour à assistance-redmine@coopdev.fr avant mise en production" + ".\n\n\n"
+        if tp_file_exists:
+            tp_f = open(tp_file,"r")
+            body = body + "Evolutions des applications tierces :\n\n" + tp_f.read()
+            tp_f.close()
+        if od_file_exists:
+            od_f = open(od_file,"r")
+            body = body + "Evolutions d'odoo :\n\n" + od_f.read()
+            tp_f.close()
+        body = "\n\nCordialement.\n\n(Ceci est un mail automatique.)"
+    msg.set_content(body)
+
+    print("don't send !!")
+    return
+
+    s = smtplib.SMTP('smtp-relay.sendinblue.com', port=587)
+    s.login("brevo@ksuite-coopdev.fr", ci_secrets.mail_server_password)
+    s.send_message(msg)
+    s.quit()
+    print("MAIL : envoi effectué")
+
+if __name__ == '__main__':
+    arg = ""
+    if len(sys.argv) > 3:
+        arg1 = sys.argv[1]
+        arg2 = sys.argv[2]
+        arg3 = sys.argv[3]
+    run(arg1,arg2,arg3)

scripts/src/supported_coops.py

+import sys
+from ci_secrets import coops_mails
+
+def is_coop_supported(coop):
+    return coop in coops_mails and coops_mails[coop]
+
+
+def is_instance_supported(instance):
+    return instance == "preprod"# or instance == "prod" ; only preprod is allowed for the moment
+
+
+def get_mail(coop):
+    return coops_mails[coop]
+
+
+if __name__ == '__main__':
+    coop = ""
+    instance = ""
+    if len(sys.argv) == 3:
+        coop = sys.argv[1]
+        instance = sys.argv[2]
+    if is_coop_supported(coop) and is_instance_supported(instance):
+        sys.exit(0)
+    else:
+        sys.exit(1)

scripts/src/update_code.sh

+#This script prepares files to send an email and update code
+update_code() {
+  coop=$1
+  instance=$2
+  repo=$3
+  now=$4
+
+  ci_dir="/home/django/third-party/scripts/src"
+  ci_data_dir="/home/django/third-party/scripts/data"
+
+
+  ###EARLY CHECKS###
+
+  if [[ ${repo} == "third-party" ]]; then
+    user="django"
+    service="django"
+  elif [[ ${repo} == "Odoo" ]]; then
+    user="odoo"
+    service="odoo"
+  else
+    echo "Incorrect repo name : ${repo}"
+    python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : incorrect repo name"
+    exit 1
+  fi
+
+  cd /home/${user}/${repo}
+  su ${user} -c "git fetch --all"
+
+  branch_found_str=$( su ${user} -c "git ls-remote origin ${coop}_prod" ) #seems that git ls-remote is not "failing" so we need to check output
+  if [ -z "${branch_found_str}" ]; then
+    #branch prod does not exist, cannot go on
+    echo "${coop} ${instance} ${repo}: branch ${coop}_prod does not exist"
+    python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : branch ${coop}_prod does not exist"
+    exit 1
+  fi
+
+
+  ###COMPARE BRANCHES AND DECIDE IF WE NEED TO CREATE A FILE CONTAINING NEW COMMITS###
+
+  new_commits="$( git log origin/${coop}_prod...origin/dev_cooperatic --no-merges --oneline )"
+  if [ -z "${new_commits}" ]; then
+    echo "${coop} ${instance} ${repo} : no new commits"
+    #As there are no new commits in prod, we don't want to send email for this repo.
+    #To indicate that, we do not create any ci_diff file at current time.
+  else
+    if ! test -f "${ci_data_dir}/ci_diff_${repo}_last.txt"; then
+      touch "${ci_data_dir}/ci_diff_${repo}_last.txt"
+    fi
+
+    echo "${new_commits}" > "${ci_data_dir}/ci_diff_${repo}_${now}.txt"
+
+    #See if commit difference is the same than last time
+    if cmp "${ci_data_dir}/ci_diff_${repo}_last.txt" "${ci_data_dir}/ci_diff_${repo}_${now}.txt"; then
+      #As there are no new commits in prod, we don't want to send email for this repo.
+      #To indicate that, we delete just created ci_diff file at current time.
+      rm "${ci_data_dir}/ci_diff_${repo}_${now}.txt"
+      echo "${coop} ${instance} ${repo} : ci_diff_${repo} files are identical"
+    else
+      #We have created new ci_diff.
+      #Replace old ci_diff_last by new one
+      rm "${ci_data_dir}/ci_diff_${repo}_last.txt"
+      cp "${ci_data_dir}/ci_diff_${repo}_${now}.txt" "${ci_data_dir}/ci_diff_${repo}_last.txt"
+    fi
+  fi
+
+
+  ###UPDATE CODE (DO IT ANYWAY AS IT IS ONLY USED ON PREPROD FOR NOW)###
+
+  #If repo is third-party, make sure config file exists as we need it later
+  if [[ ${repo} == "third-party" ]]; then
+    if ! test -f "coops_configurations/config_${coop}.py"; then
+      echo "${coop} ${instance} ${repo}: coops_configurations/config_${coop}.py does not exists"
+      python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : coops_configurations/config_${coop}.py does not exists"
+      exit 1
+    fi
+  fi
+
+  /etc/init.d/${service} stop #TODO : on prod, make sure some changes are required before stopping / restarting service
+
+  #Stash mess, then checkout proper branch and pull
+  #Stash + checkout + pull seems a good way to get rid of mess without deleting it
+  su ${user} -c "git stash"
+  #maybe some user changed branch so we need to get back on dev_cooperatic
+  if ! su ${user} -c "git checkout dev_cooperatic"; then
+    #handling cases when git checkout branch fails
+    #We can not continue but we have stopped service already, let's restart it before exiting
+    /etc/init.d/${service} start
+    echo "${coop} ${instance} ${repo}: could not git checkout dev_cooperatic"
+    python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : could not checkout dev_cooperatic"
+    exit 1
+  fi
+
+  if [[ ${repo} == "Odoo" ]]; then
+    echo "git diff --name-only dev_cooperatic..origin/dev_cooperatic"
+    updated_filepaths="$( git diff --name-only dev_cooperatic..origin/dev_cooperatic )"
+    echo "${updated_filepaths}"
+  fi
+
+  if ! su ${user} -c "git pull"; then
+    #handling cases when git pull branch fails
+    #We can not continue but we have stopped service already, let's restart it before exiting
+    /etc/init.d/${service} start
+    echo "${coop} ${instance} ${repo}: could not git pull"
+    python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : could not git pull"
+    exit 1
+  fi
+
+  if [[ ${repo} == "third-party" ]]; then
+    #Replace config file by proper one only if required
+    if ! cmp "outils/config.py" "coops_configurations/config_${coop}.py"; then
+      #Config files are different, go for it
+      mv "outils/config.py" "${ci_data_dir}/ci_old_cfg_file_${now}.py"
+      cp "coops_configurations/config_${coop}.py" "outils/config.py"
+    else
+      echo "config files are identical"
+    fi
+    /etc/init.d/django start
+  else
+    #oop on updated_filenames, for each filename, look for the directory of __openerp__.py file
+    #if found, just store the parent directory
+    SAVEIFS=$IFS
+    IFS=$'\n'
+    if [[ ${updated_filepaths} == "" ]]; then
+      updated_filepaths_array=()
+    else
+      read -rd '' -a updated_filepaths_array <<<"${updated_filepaths}"
+    fi
+
+    IFS='/'
+    l=${#updated_filepaths_array[@]}
+
+    declare -A modules #modules to update
+    modules_str=""
+
+    for (( i=0; i<l; i++ ));
+    do
+      read -rd '' -a one_path_array <<<"${updated_filepaths_array[$i]}"
+      j=0
+      accu_path="${one_path_array[0]}"
+      pathlen=${#one_path_array[@]}
+      while [[ $j -lt ${pathlen} ]] && ! test -f "${accu_path}/__openerp__.py"; do
+        ((j++))
+        accu_path="${accu_path}/${one_path_array[j]}"
+      done
+      #Update key of modules if (we have found openerp.py (j<path) + module has not been added already),
+      #so that we do not add twice same module in modules_str
+      #and add module in modules_str
+      if [[ $j -lt ${pathlen} ]] && [[ ! -v modules["${one_path_array[$j]}"] ]]; then
+        modules["${one_path_array[$j]}"]=
+        if [[ ${modules_str} == "" ]]; then
+          modules_str="${one_path_array[$j]}"
+        else
+          modules_str="${modules_str},${one_path_array[$j]}"
+        fi
+      fi
+    done
+
+    IFS=$SAVEIFS
+
+    echo "${modules_str}"
+
+    restart_odoo_cmd="/home/odoo/Odoo/odoo/odoo.py -c /home/odoo/odoo-server.conf"
+    if [[ -n "${modules_str}" ]]; then
+      echo "restart odoo service with module update"
+      su odoo -c "${restart_odoo_cmd} -u${modules_str} --stop-after-init" #TODO : make sure this is working fine
+    fi
+  fi
+
+  echo "Starting ${service} service"
+  /etc/init.d/${service} start
+}
\ No newline at end of file