Commit c636b997 by Yvon Kerdoncuff

Merge branch 'CI_for_mutualisation' into 'dev_cooperatic'

CI scripts for mutualisation : v0 (only for preprods)

See merge request !251
parents 732a5f85 28cae9c7
Pipeline #3140 failed with stage
in 1 minute 6 seconds
...@@ -20,3 +20,4 @@ shop/errors.log ...@@ -20,3 +20,4 @@ shop/errors.log
members/settings.json members/settings.json
.DS_Store .DS_Store
exec.*.log exec.*.log
scripts/src/ci_secrets.py
\ No newline at end of file
mail_server_password = "change me !"
coops_mails = {
'lacagette': 'change me !',
'supercafoutch': 'change me !',
'graoucoop': 'change me !'
}
coops_preprod_urls = {
'lacagette': 'https://preprod.odoo.lacagette-coop.fr',
'supercafoutch': 'https://preprod.odoo.supercafoutch.cooperatic.fr/',
'graoucoop': 'https://preprod.odoo.graoucoop.cooperatic.fr'
}
\ No newline at end of file
#!/bin/bash
# $1 : project name
# $2 : prod or preprod (only preprod is supported for now)
ci_dir="/home/django/third-party/scripts/src"
ci_data_dir="/home/django/third-party/scripts/data"
if ! python3 "${ci_dir}/supported_coops.py" "${1}" "${2}"; then
echo "arguments : ${1} ${2} not supported"
python3 "${ci_dir}/send_error_mail.py" "arguments : ${1} ${2} not supported"
exit 1
fi
now=$( date +%Y%m%d_%H%M%S )
source "${ci_dir}/update_code.sh"
repo="third-party"
echo "START PROCESS" "${1}" "${2}" "${repo}" "${now}"
update_code "${1}" "${2}" "${repo}" "${now}" || exit 1
echo "DONE"
repo="Odoo"
echo "START PROCESS" "${1}" "${2}" "${repo}" "${now}"
update_code "${1}" "${2}" "${repo}" "${now}" || exit 1
echo "DONE"
python3 "${ci_dir}/send_mail.py" "${ci_data_dir}/ci_diff_third-party_${now}.txt" "${ci_data_dir}/ci_diff_Odoo_${now}.txt" "${1}"
\ No newline at end of file
//Ces notes ont été prises lors de la conversion de l\'architecture cagette préprod vers la nouvelle architecture.
//Elles peuvent servir à faire de même sur la prod !
//USERS, HOME DIRS, FILES, RIGHTS
useradd -m odoo
usermod -s /bin/bash odoo
useradd -m django
usermod -s /bin/bash django
cd /home/lacagette
mv Odoo odoo-server.conf odoo-server.log odoo-server.log.1 odoo-server.log.2 scripts .local /home/odoo/.
//un nouveau odoo-server.log a été créé automatiquement, il faudra gérer ça
mv -f third-party cronlogs .virtualenvs /home/django/.
mkdir /home/odoo/cronlogs
cp -r /home/odoo/scripts /home/django/scripts
chown -R odoo:odoo /home/odoo
chown -R django:django /home/django
chown -R django:django /shared_dir/django_static
//PYTHON VENV
cd /home/django/.virtualenvs
rm -rf third-party .gitignore bin lib pyvenv.cfg
source /usr/local/bin/virtualenvwrapper.sh
mkvirtualenv -p python3.8 -r /home/django/third-party/requirements.txt third-party
//PSQL
//key to understand : "A user is a role with the ability to log in (the role has the LOGIN attribute)."
su postgres
psql
create role odoo login;
grant lacagette to odoo;
alter database lacagette owner to odoo;
select pg_terminate_backend(pid) from pg_stat_activity where pid <> pg_backend_pid() and datname = 'lacagette';
alter database lacagette rename to odoo;
exit
exit
//CONFIGS
sed -i -e 's/\/lacagette\//\/odoo\//g' /home/odoo/odoo-server.conf
sed -i -e 's/db_name = lacagette/db_name = odoo/g' /home/odoo/odoo-server.conf
sed -i -e 's/db_user = lacagette/db_user = odoo/g' /home/odoo/odoo-server.conf
sed -i -e 's/dbfilter = lacagette/dbfilter = odoo/g' /home/odoo/odoo-server.conf
sed -i -e "s/'lacagette'/'odoo'/g" /home/django/third-party/outils/settings_secret.py
sed -i -e 's/lacagette/django/g' /etc/scripts/django.sh
sed -i -e 's/lacagette/django/g' /etc/init.d/django
sed -i -e 's/lacagette/odoo/g' /etc/init.d/odoo
//CRONS
crontab -l -u lacagette | grep -v "sync_files.sh" | crontab -u django -
crontab -l -u lacagette | grep "sync_files.sh" | crontab -u odoo -
crontab -r -u lacagette
//FILESTORE ISSUE
su postgres
psql
\c odoo
delete from ir_attachment where mimetype in ('text/css','application/javascript');
//puis deux restarts nécessaires... de odoo
restart django.
import smtplib
import sys
from email.message import EmailMessage
import ci_secrets
def run(body):
msg = EmailMessage()
msg['Subject'] = "CoopdevCI : script error"
msg['From'] = "brevo@ksuite-coopdev.fr"
msg['To'] = "yvon.kerdoncuff@gmail.com"
msg.set_content(body)
s = smtplib.SMTP('smtp-relay.sendinblue.com', port=587)
s.login("brevo@ksuite-coopdev.fr", ci_secrets.mail_server_password)
s.send_message(msg)
s.quit()
if __name__ == '__main__':
arg = ""
if len(sys.argv) > 1:
arg = sys.argv[1]
run(arg)
import smtplib
import sys
from email.message import EmailMessage
import ci_secrets
import os
def run(tp_file,od_file,coop):
msg = EmailMessage()
msg['Subject'] = "Coopdev : évolutions à tester"
msg['From'] = "brevo@ksuite-coopdev.fr"
msg['To'] = ci_secrets.coops_mails[coop]
msg['CC'] = "assistance-redmine@coopdev.fr"
msg['Reply-to'] = "assistance-redmine@coopdev.fr"
tp_file_exists = os.path.exists(tp_file)
od_file_exists = os.path.exists(od_file)
if not tp_file_exists and not od_file_exists:
print("MAIL : rien à envoyer")
return
else:
body = "Cher client, chère cliente,\n"
body = body + "Les évolutions suivantes ont été mises en ligne sur " + ci_secrets.coops_preprod_urls[coop] + ".\n"
body = body + "Merci de tester leur fonctionnement et nous faire un retour à assistance-redmine@coopdev.fr avant mise en production" + ".\n\n\n"
if tp_file_exists:
tp_f = open(tp_file,"r")
body = body + "Evolutions des applications tierces :\n\n" + tp_f.read()
tp_f.close()
if od_file_exists:
od_f = open(od_file,"r")
body = body + "Evolutions d'odoo :\n\n" + od_f.read()
tp_f.close()
body = "\n\nCordialement.\n\n(Ceci est un mail automatique.)"
msg.set_content(body)
print("don't send !!")
return
s = smtplib.SMTP('smtp-relay.sendinblue.com', port=587)
s.login("brevo@ksuite-coopdev.fr", ci_secrets.mail_server_password)
s.send_message(msg)
s.quit()
print("MAIL : envoi effectué")
if __name__ == '__main__':
arg = ""
if len(sys.argv) > 3:
arg1 = sys.argv[1]
arg2 = sys.argv[2]
arg3 = sys.argv[3]
run(arg1,arg2,arg3)
import sys
from ci_secrets import coops_mails
def is_coop_supported(coop):
return coop in coops_mails and coops_mails[coop]
def is_instance_supported(instance):
return instance == "preprod"# or instance == "prod" ; only preprod is allowed for the moment
def get_mail(coop):
return coops_mails[coop]
if __name__ == '__main__':
coop = ""
instance = ""
if len(sys.argv) == 3:
coop = sys.argv[1]
instance = sys.argv[2]
if is_coop_supported(coop) and is_instance_supported(instance):
sys.exit(0)
else:
sys.exit(1)
#This script prepares files to send an email and update code
update_code() {
coop=$1
instance=$2
repo=$3
now=$4
ci_dir="/home/django/third-party/scripts/src"
ci_data_dir="/home/django/third-party/scripts/data"
###EARLY CHECKS###
if [[ ${repo} == "third-party" ]]; then
user="django"
service="django"
elif [[ ${repo} == "Odoo" ]]; then
user="odoo"
service="odoo"
else
echo "Incorrect repo name : ${repo}"
python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : incorrect repo name"
exit 1
fi
cd /home/${user}/${repo}
su ${user} -c "git fetch --all"
branch_found_str=$( su ${user} -c "git ls-remote origin ${coop}_prod" ) #seems that git ls-remote is not "failing" so we need to check output
if [ -z "${branch_found_str}" ]; then
#branch prod does not exist, cannot go on
echo "${coop} ${instance} ${repo}: branch ${coop}_prod does not exist"
python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : branch ${coop}_prod does not exist"
exit 1
fi
###COMPARE BRANCHES AND DECIDE IF WE NEED TO CREATE A FILE CONTAINING NEW COMMITS###
new_commits="$( git log origin/${coop}_prod...origin/dev_cooperatic --no-merges --oneline )"
if [ -z "${new_commits}" ]; then
echo "${coop} ${instance} ${repo} : no new commits"
#As there are no new commits in prod, we don't want to send email for this repo.
#To indicate that, we do not create any ci_diff file at current time.
else
if ! test -f "${ci_data_dir}/ci_diff_${repo}_last.txt"; then
touch "${ci_data_dir}/ci_diff_${repo}_last.txt"
fi
echo "${new_commits}" > "${ci_data_dir}/ci_diff_${repo}_${now}.txt"
#See if commit difference is the same than last time
if cmp "${ci_data_dir}/ci_diff_${repo}_last.txt" "${ci_data_dir}/ci_diff_${repo}_${now}.txt"; then
#As there are no new commits in prod, we don't want to send email for this repo.
#To indicate that, we delete just created ci_diff file at current time.
rm "${ci_data_dir}/ci_diff_${repo}_${now}.txt"
echo "${coop} ${instance} ${repo} : ci_diff_${repo} files are identical"
else
#We have created new ci_diff.
#Replace old ci_diff_last by new one
rm "${ci_data_dir}/ci_diff_${repo}_last.txt"
cp "${ci_data_dir}/ci_diff_${repo}_${now}.txt" "${ci_data_dir}/ci_diff_${repo}_last.txt"
fi
fi
###UPDATE CODE (DO IT ANYWAY AS IT IS ONLY USED ON PREPROD FOR NOW)###
#If repo is third-party, make sure config file exists as we need it later
if [[ ${repo} == "third-party" ]]; then
if ! test -f "coops_configurations/config_${coop}.py"; then
echo "${coop} ${instance} ${repo}: coops_configurations/config_${coop}.py does not exists"
python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : coops_configurations/config_${coop}.py does not exists"
exit 1
fi
fi
/etc/init.d/${service} stop #TODO : on prod, make sure some changes are required before stopping / restarting service
#Stash mess, then checkout proper branch and pull
#Stash + checkout + pull seems a good way to get rid of mess without deleting it
su ${user} -c "git stash"
#maybe some user changed branch so we need to get back on dev_cooperatic
if ! su ${user} -c "git checkout dev_cooperatic"; then
#handling cases when git checkout branch fails
#We can not continue but we have stopped service already, let's restart it before exiting
/etc/init.d/${service} start
echo "${coop} ${instance} ${repo}: could not git checkout dev_cooperatic"
python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : could not checkout dev_cooperatic"
exit 1
fi
if [[ ${repo} == "Odoo" ]]; then
echo "git diff --name-only dev_cooperatic..origin/dev_cooperatic"
updated_filepaths="$( git diff --name-only dev_cooperatic..origin/dev_cooperatic )"
echo "${updated_filepaths}"
fi
if ! su ${user} -c "git pull"; then
#handling cases when git pull branch fails
#We can not continue but we have stopped service already, let's restart it before exiting
/etc/init.d/${service} start
echo "${coop} ${instance} ${repo}: could not git pull"
python3 "${ci_dir}/send_error_mail.py" "${coop} ${instance} ${repo} : could not git pull"
exit 1
fi
if [[ ${repo} == "third-party" ]]; then
#Replace config file by proper one only if required
if ! cmp "outils/config.py" "coops_configurations/config_${coop}.py"; then
#Config files are different, go for it
mv "outils/config.py" "${ci_data_dir}/ci_old_cfg_file_${now}.py"
cp "coops_configurations/config_${coop}.py" "outils/config.py"
else
echo "config files are identical"
fi
/etc/init.d/django start
else
#oop on updated_filenames, for each filename, look for the directory of __openerp__.py file
#if found, just store the parent directory
SAVEIFS=$IFS
IFS=$'\n'
if [[ ${updated_filepaths} == "" ]]; then
updated_filepaths_array=()
else
read -rd '' -a updated_filepaths_array <<<"${updated_filepaths}"
fi
IFS='/'
l=${#updated_filepaths_array[@]}
declare -A modules #modules to update
modules_str=""
for (( i=0; i<l; i++ ));
do
read -rd '' -a one_path_array <<<"${updated_filepaths_array[$i]}"
j=0
accu_path="${one_path_array[0]}"
pathlen=${#one_path_array[@]}
while [[ $j -lt ${pathlen} ]] && ! test -f "${accu_path}/__openerp__.py"; do
((j++))
accu_path="${accu_path}/${one_path_array[j]}"
done
#Update key of modules if (we have found openerp.py (j<path) + module has not been added already),
#so that we do not add twice same module in modules_str
#and add module in modules_str
if [[ $j -lt ${pathlen} ]] && [[ ! -v modules["${one_path_array[$j]}"] ]]; then
modules["${one_path_array[$j]}"]=
if [[ ${modules_str} == "" ]]; then
modules_str="${one_path_array[$j]}"
else
modules_str="${modules_str},${one_path_array[$j]}"
fi
fi
done
IFS=$SAVEIFS
echo "${modules_str}"
restart_odoo_cmd="/home/odoo/Odoo/odoo/odoo.py -c /home/odoo/odoo-server.conf"
if [[ -n "${modules_str}" ]]; then
echo "restart odoo service with module update"
su odoo -c "${restart_odoo_cmd} -u${modules_str} --stop-after-init" #TODO : make sure this is working fine
fi
fi
echo "Starting ${service} service"
/etc/init.d/${service} start
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment