说明

Nginx广泛被作为web服务器使用，自身支持热更新操作。如我们在宿主机上通过yum/apt/编译安装/二进制安装nginx，在我们修改配置文件之后，执行nginx -s reload命令可以不停服务重新加载配置。

对于使用docker部署的nginx，我们也可以使用docker exec -it nginx-container service nginx reload来完成修改后的配置文件重新加载，但无论对于docker/k8s这样过多的人工干预还是很痛苦的。

本文将使用sidecar(边车)方案来完成nginx的自动热更新，同时通过演示k8s部署nginx下载服务器来更好的帮助大家理解。

环境说明

# k8s version
v1.18.8-eks
# linux version
Red Hat 7.3.1-6
# docker version
19.03.13-ce

部署过程

部署过程主要分两部分，一是nginx热更新镜像build，二是k8s部署nginx下载服务器。当然，镜像也可直接使用我分享的~

nginx热更新镜像build

Dockerfile

Sidecar容器，nginx-reloader镜像的Dockerfile如下：

# cat Dockerfile

FROM golang:1.12.0 as build
RUN go get github.com/fsnotify/fsnotify
RUN go get github.com/shirou/gopsutil/process
RUN mkdir -p /go/src/app
ADD main.go /go/src/app/
WORKDIR /go/src/app
RUN CGO_ENABLED=0 GOOS=linux go build -a -o nginx-reloader .
# main image
FROM nginx:1.14.2-alpine
COPY --from=build /go/src/app/nginx-reloader /
CMD ["/nginx-reloader"]

nginx热更新功能是用一个叫main.go的go脚本实现的：

# cat main.go

package main

import (
    "log"
    "os"
    "path/filepath"
    "syscall"

    "github.com/fsnotify/fsnotify"
    proc "github.com/shirou/gopsutil/process"
)

const (
    nginxProcessName = "nginx"
    defaultNginxConfPath = "/etc/nginx"
    watchPathEnvVarName = "WATCH_NGINX_CONF_PATH"
)

var stderrLogger = log.New(os.Stderr, "error: ", log.Lshortfile)
var stdoutLogger = log.New(os.Stdout, "", log.Lshortfile)

func getMasterNginxPid() (int, error) {
    processes, processesErr := proc.Processes()
    if processesErr != nil {
        return 0, processesErr
    }

    nginxProcesses := map[int32]int32{}

    for _, process := range processes {
        processName, processNameErr := process.Name()
        if processNameErr != nil {
            return 0, processNameErr
        }

        if processName == nginxProcessName {
            ppid, ppidErr := process.Ppid()

            if ppidErr != nil {
                return 0, ppidErr
            }

            nginxProcesses[process.Pid] = ppid
        }
    }

    var masterNginxPid int32

    for pid, ppid := range nginxProcesses {
        if ppid == 0 {
            masterNginxPid = pid

            break
        }
    }

    stdoutLogger.Println("found master nginx pid:", masterNginxPid)

    return int(masterNginxPid), nil
}

func signalNginxReload(pid int) error {
    stdoutLogger.Printf("signaling master nginx process (pid: %d) -> SIGHUP\n", pid)
    nginxProcess, nginxProcessErr := os.FindProcess(pid)

    if nginxProcessErr != nil {
        return nginxProcessErr
    }

    return nginxProcess.Signal(syscall.SIGHUP)
}

func main() {
    watcher, watcherErr := fsnotify.NewWatcher()
    if watcherErr != nil {
        stderrLogger.Fatal(watcherErr)
    }
    defer watcher.Close()

    done := make(chan bool)
    go func() {
        for {
            select {
            case event, ok := <-watcher.Events:
                if !ok {
                    return
                }

                if event.Op&fsnotify.Create == fsnotify.Create {
                    if filepath.Base(event.Name) == "..data" {
                        stdoutLogger.Println("config map updated")

                        nginxPid, nginxPidErr := getMasterNginxPid()
                        if nginxPidErr != nil {
                            stderrLogger.Printf("getting master nginx pid failed: %s", nginxPidErr.Error())

                            continue
                        }

                        if err := signalNginxReload(nginxPid); err != nil {
                            stderrLogger.Printf("signaling master nginx process failed: %s", err)
                        }
                    }
                }
            case err, ok := <-watcher.Errors:
                if !ok {
                    return
                }
                stderrLogger.Printf("received watcher.Error: %s", err)
            }
        }
    }()

    pathToWatch, ok := os.LookupEnv(watchPathEnvVarName)
    if !ok {
        pathToWatch = defaultNginxConfPath
    }

    stdoutLogger.Printf("adding path: `%s` to watch\n", pathToWatch)

    if err := watcher.Add(pathToWatch); err != nil {
        stderrLogger.Fatal(err)
    }
    <-done
}

构建镜像

基于该Dockerfile构建nginx-reloader镜像：

docker build -t registry.cn-shanghai.aliyuncs.com/tengfeiwu/nginx-reloader:20210521 .

k8s部署nginx下载服务器

创建kube-mon命名空间

# cat ns-nginx.yml

apiVersion: v1
kind: Namespace
metadata:
  name: kube-mon

创建nginx configmap

# cat nginx-configmap.yml

apiVersion: v1
kind: ConfigMap
metadata:
  name: nginx-config
  namespace: kube-mon
data:
  nginx.conf: |
    user  root;
    worker_processes  2;
    error_log  /var/log/nginx/error.log error;
    pid        /var/run/nginx.pid;
    events {
        worker_connections  10240;
    }
    http {
      default_type  application/octet-stream;
      log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                        '$status $body_bytes_sent "$http_referer" '
                        '"$http_user_agent" "$http_x_forwarded_for"';
      access_log  /var/log/nginx/access.log  main;
      sendfile        on;
      #tcp_nopush     on;
      keepalive_timeout  65;
      #gzip  on;
      include /etc/nginx/conf.d/*.conf;
    }

创建vhost

# cat ai-download.yml 

apiVersion: v1
kind: ConfigMap
metadata:
  name: download-config
  namespace: kube-mon
data:
  ai-download.conf: |
    server {
    listen       80;                #端口 
    #server_name  localhost;        #服务名
    charset utf-8;                  #避免中文乱码
    root /tmp;                      #显示的根索引目录，注意这里要改成你自己的，目录要存在  

    location / {
        autoindex on;               #开启索引功能 
        autoindex_exact_size off;   #关闭计算文件确切大小（单位bytes），只显示大概大小（单位kb、mb、gb）  
        autoindex_localtime on;     #显示本机时间而非 GMT 时间
      }
    }

创建deployment

# cat nginx-all-reloader.yml

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  namespace: kube-mon
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      name: nginx
      labels:
        app: nginx
    spec:
      # 需打开共享进程命名空间特性
      shareProcessNamespace: true
      volumes:
        - name: nginx-config
          configMap:
            name: nginx-config
        - name: ai-config
          configMap:
            name: download-config
        - name: mongodb-efs-data
          hostPath:
            # 宿主上目录位置
            path: /mnt/data-s3-fs/mongodb-efs
            #type: Directory 
      containers:
        - name: nginx
          image: nginx
          ports:
            - name: http
              containerPort: 80
            - name: https
              containerPort: 443
          volumeMounts:
            - name: nginx-config
              mountPath: /etc/nginx/nginx.conf
              subPath: nginx.conf
              readOnly: true
            - name: ai-config
              mountPath: /etc/nginx/conf.d
              readOnly: true
            - name: mongodb-efs-data
              mountPath: /tmp
              readOnly: true

        - name: nginx-reloader
          image: registry.cn-shanghai.aliyuncs.com/tengfeiwu/nginx-reloader:20210521
          env:
            - name: WATCH_NGINX_CONF_PATH
              value: /etc/nginx/conf.d
          volumeMounts:
            - name: ai-config
              mountPath: /etc/nginx/conf.d
              readOnly: true
---
apiVersion: v1
kind: Service
metadata:
  name: nginx
  namespace: kube-mon
spec:
  selector:
    app: nginx
  type: NodePort
  ports:
    - name: http
      port: 80
      nodePort: 32080

部署应用

# 创建namespace
kubectl apply -f ns-nginx.yml
# 创建nginx configmap
kubectl apply -f nginx-configmap.yml
# 创建download配置文件，支持热更新
kubectl apply -f ai-download.yml
# 部署应用
kubectl apply -f nginx-all-reloader.yml

实现效果

浏览器访问nginx下载服务：http://ip:32080，如下：

手动修改ai-download.yml后再apply，reloader监测到configmap变化，会主动向nginx主进程发起HUP信号，实现配置热更新。