Jenkins小项目—代码测试、部署、回滚、keepalived+haproxy调度至tomcat
1.分别配置两台tomcat后端服务的java环境
1）准备jdk8压缩包
[root@tomcat jdk]# pwd
/usr/local/src/jdk
root@tomcat jdk]# ls
jdk-8u211-linux-x64.tar.gz2）解压jdk压缩包当前目录下并创建软连接
[root@tomcat jdk]# tar xvf jdk-8u211-linux-x64.tar.gz
[root@tomcat jdk]# ln -sv jdk1.8.0_211/ jdk3）配置java的环境变量并生效
[root@tomcat ~]# vim /etc/profile
……
export JAVA_HOME=/usr/local/src/jdk/jdk
export JRE_HOME=$JAVA_HOME/jre
export CLASSPATH=$JAVA_HOME/lib/:$JRE_HOME/lib
export TOMCAT_HOME=/usr/local/src/tomcat/tomcat
export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin:$TOMCAT_HOME/bin
[root@tomcat ~]# source /etc/profile
4)测试java环境
[root@tomcat ~]# echo $JAVA_HOME
/usr/local/src/jdk/jdk
[root@tomcat ~]# echo $CLASSPATH
/usr/local/src/jdk/jdk/lib/:/usr/local/src/jdk/jdk/jre/lib
[root@tomcat ~]# echo $PATH
/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/usr/local/src/jdk/jdk/bin:/usr/local/src/jdk/jdk/jre/bin:/usr/local/src/tomcat/tomcat/bin:/root/bin
[root@tomcat ~]# java -version #查看java的版本
java version "1.8.0_211"
Java(TM) SE Runtime Environment (build 1.8.0_211-b12)
Java HotSpot(TM) 64-Bit Server VM (build 25.211-b12, mixed mode)2.分别安装配置tomcat服务
1)准备tomcat二进制压缩包
[root@tomcat tomcat]# pwd
/usr/local/src/tomcat
[root@tomcat tomcat]# ls
apache-tomcat-8.5.43.tar.gz2）解压tomcat压缩文件并创建软连接
[root@tomcat tomcat]# tar xvf apache-tomcat-8.5.43.tar.gz
[root@tomcat tomcat]# ln -sv apache-tomcat-8.5.43 tomcat3）启动tomcat服务
[root@tomcat ~]# /usr/local/src/tomcat/tomcat/bin/catalina.sh start4）查看启动端口
[root@tomcat ~]# ss -tnlp
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 *:22 *:* users:(("sshd",pid=3716,fd=3))
LISTEN 0 100 127.0.0.1:25 *:* users:(("master",pid=3936,fd=13))
LISTEN 0 100 :::8009 :::* users:(("java",pid=5861,fd=54))
LISTEN 0 100 :::8080 :::* users:(("java",pid=5861,fd=49))
LISTEN 0 128 :::22 :::* users:(("sshd",pid=3716,fd=4))
LISTEN 0 100 ::1:25 :::* users:(("master",pid=3936,fd=14))
LISTEN 0 1 ::ffff:127.0.0.1:8005 :::* users:(("java",pid=5861,fd=75))5）浏览器访问测试“主tomcat服务”
6）浏览器访问测试“备tomcat-1服务”
3.分别配置两台keepalived+haproxy高可用分离调度服务
1)安装高可用服务keepalived
[root@keepalive_haproxy ~]# yum install keepalive -y2）修改keepalived配置文件
[root@keepalive_haproxy ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen@firewall.loc
failover@firewall.loc
sysadmin@firewall.loc
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id haproxy #在备份服务中的路由id设置为 ”haproxy-1“，不可相同
vrrp_skip_check_adv_addr
# vrrp_strict #禁用掉vrrp，否则只支持组播不支持单播模式
vrrp_iptables #开启不自动添加防火墙规则，避免无法访问此主机
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state MASTER #设置为主服务，在备份服务中设置为”BACKUP“，备份服务
interface eth0 #绑定的网卡
virtual_router_id 51 #实例路由id号，此id号主备服务可相同
priority 100 #优先级，备份服务优先级必须小于100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.10.23/ dev eth0 label eth0:0 #将虚拟vip绑定到本地eth0网卡并取名为eth0:0
}
unicast_src_ip 192.168.1.10 #单播源地址ip，在备份服务中设置源ip为192.168.1.11
unicast_peer{
192.168.1.11 #单播目标地址ip，在备份服务中设置目标ip为192.168.1.10
}
}3）分别启动keepalived服务
主keepalivd：
[root@keepalive_haproxy ~]# systemctl start keepalived
root@keepalive_haproxy ~]# systemctl status keepalived
● keepalived.service - LVS and VRRP High Availability Monitor
Loaded: loaded (/usr/lib/systemd/system/keepalived.service; enabled; vendor preset: disabled)
Active: active (running) since Mon 2019-08-05 18:10:00 CST; 21s ago
Process: 4313 ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS (code=exited, status=0/SUCCESS)
Main PID: 4314 (keepalived)
[root@keepalive_haproxy ~]# ip a
……
eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:36:53:00 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.10/24 brd 192.168.1.255 scope global eth0
valid_lft forever preferred_lft forever
inet 192.168.10.23/0 scope global eth0:0 #绑定的虚拟vip
valid_lft forever preferred_lft forever
……备keepalivd：
[root@keepalive_haproxy ~]# systemctl status keepalived
● keepalived.service - LVS and VRRP High Availability Monitor
Loaded: loaded (/usr/lib/systemd/system/keepalived.service; enabled; vendor preset: disabled)
Active: active (running) since Mon 2019-08-05 17:32:01 CST; 40min ago
Process: 3712 ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS (code=exited, status=0/SUCCESS)
Main PID: 3853 (keepalived)
[root@keepalive_haproxy ~]# ip a #没有看到虚拟vip，当主服务挂掉，虚拟vip会自动漂移到此主机
……
eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:c4:e2:07 brd ff:ff:ff:ff:ff:ff
inet 192.168.1.11/24 brd 192.168.1.255 scope global eth0
……4）分别配置两台调度服务内核参数
[root@keepalive_haproxy ~]# vim /etc/sysctl.conf
……
net.ipv4.ip_nonlocal_bind = 1 #开启非本地ip绑定，避免haproxy无法绑定非本机ip
net.ipv4.ip_forward = 1 #开启路由转发功能5）生效内核参数
[root@keepalive_haproxy ~]# sysctl -p
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 16）分别编译安装好haproxy，以下为编译安装后的路径
[root@keepalive_haproxy haproxy]# pwd
/usr/local/src/haproxy
[root@keepalive_haproxy haproxy]# ls
doc sbin share7）再修改配置文件，两台服务配置文件必须保持相同
[root@keepalive_haproxy ~]# vim /etc/haproxy/haproxy.cfg
Global
maxconn 100000 #每个进程并发最大连接数
chroot /usr/local/src/haproxy #锁定 运行的目录
#stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin
stats socket /usr/local/src/haproxy/haproxy.sock mode 600 level admin #自定义sock
文件路径，此路径下haproxy启动用户必须有权限创建haproxy.sock文件，否则服务无法
启动，此sock文件为支持动态下线后端服务功能，也可注释掉不创建sock文件
uid 88 #执行haproxy的用户身份
gid 88 #所属的组
daemon
nbproc 2 #开启的线程数
cpu-map 1 0 #绑定到cup的第0号核心
cpu-map 2 1 #绑定到cup的第1号核心
pidfile /run/haproxy.pid #pid文件路径
log 127.0.0.1 local3 info #定义全局syslog
defaults #默认设置，为前端、后端及listen默认设置
option http-keep-alive
option forwardfor #ip透传
maxconn 100000
mode http
timeout connect 300000ms
timeout client 300000ms
timeout server 300000ms
listen stats #开启监听状态页
mode http #http协议
bind 0.0.0.0:9999 #状态页访绑定的端口
stats enable #开启状态页
log global #全局日志
stats uri /haproxy-status #状态也路径
stats auth admin:123456 #状态页登录的用户名及密码
listen web_port #监听的服务
bind 192.168.10.23:80 #绑定的虚拟vip及端口，当外网访问此虚拟vip时会自动调度到后端服务
mode http #http协议
balance roundrobin #调度算法 roundrobin动态轮询
log global #全局日志
server 192.168.1.20 192.168.1.20:8080 check inter 3000 fall 2 rise 5 #调度的后端服务
server 192.168.1.21 192.168.1.21:8080 check inter 3000 fall 2 rise 5 #调度的后端服务8）创建haproxy启动用户
[root@keepalive_haproxy haproxy]# useradd -r -s /sbin/nologin haproxy -u 889）分别创建haproxy启动脚本
[root@keepalive_haproxy haproxy]# vim /usr/lib/systemd/system/haproxy.service
[Unit]
Description=HAproxy Load Balancer
After=syslog.target network.target
[Service]
ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q
ExecStart=/usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid
ExecStop=/bin/kill -USR2 $MAINPID
[Install]
WantedBy=multi-user.target10）分别启动haproxy服务
[root@keepalive_haproxy haproxy]# systemctl start haproxy
root@keepalive_haproxy haproxy]# systemctl status haproxy
● haproxy.service - HAproxy Load Balancer
Loaded: loaded (/usr/lib/systemd/system/haproxy.service; enabled; vendor preset: disabled)
Active: active (running) since Mon 2019-08-05 17:31:48 CST; 1h 25min ago
Process: 3716 ExecStartPre=/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -c -q (code=exited, status=0/SUCCESS)
Main PID: 3769 (haproxy)
[root@keepalive_haproxy haproxy]# ss -tnlp
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 *:9999 *:* users:(("haproxy",pid=3828,fd=5),("haproxy",pid=3827,fd=5))
LISTEN 0 128 192.168.10.23:80 *:* users:(("haproxy",pid=3828,fd=7),("haproxy",pid=3827,fd=7))
……11)haproxy状态页访问
12）浏览器访问调度服务，成功调度到后端服务
4.创建Jenkins的执行脚本，用以实现通过Jenkins的选项参数来自动测试、部署、回滚代码（事先搭建好jenkins、gitlab、sonaqube等服务，其中jenkins要安装scanner扫描器）
1）自定义创建指定的jenkins服务工作目录
[root@jenkins]# mkdir /data/jenkins/worker -pv2）jenkins服务器脚本的保存路径
[root@jenkins jenkins]# pwd
/data/jenkins3）jenkins服务器编辑脚本
[root@jenkins jenkins]# vim project.sh
#!/bin/bash
#jenkins参数选项
time=`date +%Y-%m-%d_%H-%M-%S`
method=$1
group=$2
branch=$3
#后端tomcat服务ip地址组
ip_value(){
if [[ $group == "group1" ]];then
ip_list="192.168.1.20"
echo ${ip_list}
elif [[ $group == "group2" ]];then
ip_list="192.168.1.21"
echo ${ip_list}
ssh root@192.168.1.10 "echo "enable server web_port/192.168.1.20" | socat stdio /usr/local/src/haproxy/haproxy.sock"
ssh root@192.168.1.11 "echo "enable server web_port/192.168.1.20" | socat stdio /usr/local/src/haproxy/haproxy.sock"
elif [[ $group == "group3" ]];then
ip_list="192.168.1.20 192.168.1.21"
echo ${ip_list}
fi
}
#代码先部署至Jenkins服务端
code_deploy(){
cd /data/jenkins/worker
rm -rf ./*
git clone -b $branch git@192.168.1.30:jie/web-page.git
}
#代码测试
code_test(){
cd /data/jenkins/worker/web-page
cat > sonar-project.properties <<eof
sonar.projectKey=one123456
sonar.projectName=code-test
sonar.projectVersion=1.0
sonar.sources=./
sonar.language=php
sonar.sourceEncoding=UTF-8
eof
/data/scanner/sonar-scanner/bin/sonar-scanner
}
#代码压缩
code_compress(){
cd /data/jenkins/worker/
rm -f web-page/sonar-project.properties
tar czvf code-tar.gz web-page
}
#调度器剥离后端服务
haproxy_down(){
for ip in ${ip_list};do
echo $ip
ssh root@192.168.1.10 "echo "disable server web_port/${ip}"|socat stdio /usr/local/src/haproxy/haproxy.sock"
ssh root@192.168.1.11 "echo "disable server web_port/${ip}"|socat stdio /usr/local/src/haproxy/haproxy.sock"
done
}
#后端服务下线
backend_stop(){
for ip in ${ip_list};do
echo $ip
ssh root@$ip "source /etc/profile && /usr/local/src/tomcat/tomcat/bin/catalina.sh stop"
done
}
#部署代码到后端服务站点
scp_backend(){
for ip in ${ip_list};do
echo $ip
scp /data/jenkins/worker/code-tar.gz root@${ip}:/usr/local/src/tomcat/tomcat/web-code/${time}-code-tar.gz
ssh root@${ip} "tar xvf /usr/local/src/tomcat/tomcat/web-code/${time}-code-tar.gz -C /usr/local/src/tomcat/tomcat/webapps"
done
}
#启动后端服务
backend_start(){
for ip in ${ip_list};do
echo $ip
ssh root@$ip "source /etc/profile && /usr/local/src/tomcat/tomcat/bin/catalina.sh start"
sleep 6
done
}
#测试访问后端服务
backend_test(){
for ip in ${ip_list};do
echo $ip
status_code=`curl -I -s -m 6 -o /dev/null -w %{http_code} http://${ip}:8080`
if [ ${status_code} -eq 200 ];then
echo "访问测试成功，后端代码部署成功"
if [[ $ip == "192.168.1.21" ]];then
ssh root@192.168.1.10 "echo "enable server web_port/${ip}" | socat stdio /usr/local/src/haproxy/haproxy.sock"
ssh root@192.168.1.11 "echo "enable server web_port/${ip}" | socat stdio /usr/local/src/haproxy/haproxy.sock"
fi
else
echo "访问测试失败，请重新部署代码至后端服务"
fi
done
}
#代码回滚
code_rollback(){
for ip in ${ip_list};do
echo $ip
last_version=`ssh root@${ip} "ls -l -t /usr/local/src/tomcat/tomcat/web-code/" | awk 'NR==3{print $NF}'`
ssh root@${ip} " tar xvf /usr/local/src/tomcat/tomcat/web-code/$last_version -C /usr/local/src/tomcat/tomcat/webapps"
done
echo "tomcat代码回滚成功，回到上一版本，下一步进行访问测试"
}
#主菜单命令
main(){
case $1 in
deploy)
ip_value;
code_deploy;
code_test;
code_compress;
haproxy_down;
backend_stop;
scp_backend;
backend_start;
backend_test;
;;
rollback)
ip_value;
haproxy_down;
backend_stop;
code_rollback;
backend_start;
backend_test;
;;
esac
}
main $1 $2 $34）再各后端创建好代码压缩文件保存路径
主tomcat：
[root@tomcat tomcat]# mkdir web-code
[root@tomcat tomcat]# pwd
/usr/local/src/tomcat/tomcat备tomcat-1：
[root@tomcat-1 tomcat]# mkdir web-code
[root@tomcat-1 tomcat]# pwd
/usr/local/src/tomcat/tomcat5）再jenkins服务设置好免密秘钥登录各服务
[root@jenkins jenkins]# ssh-copy-id 192.168.1.10
[root@jenkins jenkins]# ssh-copy-id 192.168.1.11
[root@jenkins jenkins]# ssh-copy-id 192.168.1.20
[root@jenkins jenkins]# ssh-copy-id 192.168.1.215.再gitlab服务器克隆并推送代码
1）克隆指定的develop分支代码
root@ubuntu1804:~# git clone -b develop http://192.168.1.30/jie/web-page.git
Cloning into 'web-page'...
Username for 'http://192.168.1.30': jie
Password for 'http://jie@192.168.1.30':
remote: Enumerating objects: 39, done.
remote: Counting objects: 100% (39/39), done.
remote: Compressing objects: 100% (22/22), done.
remote: Total 39 (delta 4), reused 27 (delta 4)
Unpacking objects: 100% (39/39), done.2）查看克隆的所包含的代码文件
root@ubuntu1804:~# ls web-page/
index.html Math.php3）修改代文件
root@ubuntu1804:~/web-page# cat index.html
<h1>welcome to tomcat page</h1>
<h3>simple-version v1</h3>4）推送v1版代码至gitlab代码库
root@ubuntu1804:~/web-page# git add ./*
root@ubuntu1804:~/web-page# git commit -m 'v1'
[develop d0dd713] v1
1 file changed, 2 insertions(+), 2 deletions(-)
root@ubuntu1804:~/web-page# git push
Username for 'http://192.168.1.30': jie
Password for 'http://jie@192.168.1.30':
Counting objects: 3, done.
Delta compression using up to 4 threads.
Compressing objects: 100% (3/3), done.
Writing objects: 100% (3/3), 316 bytes | 316.00 KiB/s, done.
Total 3 (delta 0), reused 0 (delta 0)
remote:
remote: To create a merge request for develop, visit:
remote: http://192.168.1.30/jie/web-page/merge_requests/new?merge_request%5Bsource_branch%5D=develop
remote:
To http://192.168.1.30/jie/web-page.git
c10f5bf..d0dd713 develop -> develop6.jenkins的配置文件修改及选项参数构建
1）创建一个项目code-test
2）配置此项目的configure文件，添加选项参数、字符参数且与脚本文件中的选项相对应
3）配置jenkins的shell脚本命令，此脚本实现代码的测试、部署以及 回滚
4）保存以上配置，然后部署第一组后端服务主tomcat
5）控制台输出信息
6）直接浏览器访问主tomcat服务验证是否部署成功
7）再部署第二组后端服务备tomcat-1
8）控制台输出部署成功信息
9）分别查看后端服务部署的相关代码文件，确定代码文件是否部署到后端服务
主tomcat服务端：
[root@tomcat tomcat]# pwd/usr/local/src/tomcat/tomcat
[root@tomcat tomcat]# ll web-code/
total 16
-rw-r--r-- 1 root root 14910 Aug 4 18:23 2019-08-04_18-23-01-code-tar.gz
[root@tomcat webapps]# pwd
/usr/local/src/tomcat/tomcat/webapps
[root@tomcat webapps]# cat web-page/index.html
<h1>welcome to tomcat page</h1>
<h3>simple-version v1</h3>备tomcat-1服务端：
[root@tomcat-1 tomcat]# pwd
/usr/local/src/tomcat/tomcat
[root@tomcat-1 tomcat]# ll web-code/
total 16
-rw-r--r-- 1 root root 14910 Aug 4 18:23 2019-08-04_18-23-01-code-tar.gz
[root@tomcat-1 webapps]# pwd
/usr/local/src/tomcat/tomcat/webapps
[root@tomcat-1 webapps]# cat web-page/index.html
<h1>welcome to tomcat page</h1>
<h3>simple-version v1</h3>10）直接浏览器访问备tomcat1服务验证是否部署成功，如下所示备tomcat-1代码也部署成功
11）最后通过浏览器haproxy调度器，成功调度到后端服务tomcat
12）代码测试结果
7.将后端服务代码升级到v2新版本
1）在gitlab服务器更新代码
root@ubuntu1804:~/web-page# cat index.html
<h1>welcome to tomcat page</h1>
<h3>enhanced-version v2--- Handled bugs on the old v1</h3>2）在gitlab服务器再次推送v2新版本代码至gitlab代码库
root@ubuntu1804:~/web-page# git add ./*
root@ubuntu1804:~/web-page# git commit -m 'v2'
[develop 2512294] v2
1 file changed, 1 insertion(+), 1 deletion(-)
root@ubuntu1804:~/web-page# git push
Username for 'http://192.168.1.30': jie
Password for 'http://jie@192.168.1.30':
Counting objects: 3, done.
Delta compression using up to 4 threads.
Compressing objects: 100% (3/3), done.
Writing objects: 100% (3/3), 344 bytes | 344.00 KiB/s, done.
Total 3 (delta 0), reused 0 (delta 0)
remote:
remote: To create a merge request for develop, visit:
remote: http://192.168.1.30/jie/web-page/merge_requests/new?merge_request%5Bsource_branch%5D=develop
remote:
To http://192.168.1.30/jie/web-page.git
d0dd713..2512294 develop -> develop3）构建参数group3，将所有后端服务全部更新，升级至v2版本
4）查看控制台执行的结果，显示部署成功
5）在各后端服务端查看更新的代码文件，检查代码是否更新，并浏览器测试访问
tomcat服务端：
[root@tomcat webapps]# cat web-page/index.html
<h1>welcome to tomcat page</h1>
<h3>enhanced-version v2--- Handled bugs on the old v1</h3>
tomcat-1服务端：
[root@tomcat-1 webapps]# cat web-page/index.html
<h1>welcome to tomcat page</h1>
<h3>enhanced-version v2--- Handled bugs on the old v1</h3>
6）再通过浏览器访问haproxy调度器服务，检查更新代码后是否成功调度到后端服务
7）代码测试结果
8.代码回滚到旧版本（若v2版本不稳定，则需要将后端服务代码回滚到旧版本v1，避免影响业务的进行）
1）构建选择rollback回滚，group3所有后端回滚
2）查看控制台输代码部署输出信息
3）查看各后端服务代码文件，检查代码是否回滚到v1旧版本，并浏览器测试各服务端
主tomcat服务端：
[root@tomcat webapps]# cat web-page/index.html
<h1>welcome to tomcat page</h1>
<h3>simple-version v1</h3>
备tomcat-1服务端：
[root@tomcat-1 webapps]# cat web-page/index.html
<h1>welcome to tomcat page</h1>
<h3>simple-version v1</h3>
4）访问haproxy调度器服务，代码回滚成功
