【Golang】使用 - 优雅重启进程

Posted by 西维蜀黍 on 2021-04-05, Last Modified on 2021-09-21

Graceful Restart

If you have a Golang HTTP service, chances are, you will need to restart it on occasion to upgrade the binary or change some configuration. And if you (like me) have been taking graceful restart for granted because the webserver took care of it, you may find this recipe very handy because with Golang you need to roll your own.

There are actually two problems that need to be solved here. First is the UNIX side of the graceful restart, i.e. the mechanism by which a process can restart itself without closing the listening socket. The second problem is ensuring that all in-progress requests are properly completed or timed-out.

值得思考的一种情况是,如果在处理这些 in-progress requests 的过程中,又有 request 来了,by right 需要将这些后来的 request处理完后,才能真正进行 restart;而如果现在处理这些后来的 request 的时候,新的 request又来了呢?so on and so forth。

In practise, 我们可能会在 Nginx,服务端代码经常需要升级,对于线上系统的升级常用的做法是,通过前端的负载均衡(如nginx)来保证升级时至少有一个服务可用,依次(灰度)升级。 而另一种更方便的方法是在应用上做热重启,直接升级应用而不停服务。

package main

import (
	"context"
	"flag"
	"fmt"
	"net"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"syscall"
)

var (
	upgrade bool
	ln      net.Listener
	server  *http.Server
)

func init() {
	flag.BoolVar(&upgrade, "upgrade", false, "user can't use this")
}

func hello(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "hello world from pid:%d, ppid: %d\n", os.Getpid(), os.Getppid())
}

func main() {
	flag.Parse()
	http.HandleFunc("/", hello)
	server = &http.Server{Addr: ":8999"}
	var err error
	if upgrade {
		fd := os.NewFile(3, "")
		ln, err = net.FileListener(fd)
		if err != nil {
			fmt.Printf("fileListener fail, error: %s\n", err)
			os.Exit(1)
		}
		fd.Close()
	} else {
		ln, err = net.Listen("tcp", server.Addr)
		if err != nil {
			fmt.Printf("listen %s fail, error: %s\n", server.Addr, err)
			os.Exit(1)
		}
	}
	go func() {
		err := server.Serve(ln)
		if err != nil && err != http.ErrServerClosed {
			fmt.Printf("serve error: %s\n", err)
		}
	}()
	setupSignal()
	fmt.Println("over")
}

func setupSignal() {
	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGUSR2, syscall.SIGINT, syscall.SIGTERM)
	sig := <-ch
	switch sig {
	case syscall.SIGUSR2:
		fmt.Println("SIGUSR2 received")
		err := forkProcess()
		if err != nil {
			fmt.Printf("fork process error: %s\n", err)
		}
		err = server.Shutdown(context.Background())
		if err != nil {
			fmt.Printf("shutdown after forking process error: %s\n", err)
		}
	case syscall.SIGINT, syscall.SIGTERM:
		signal.Stop(ch)
		close(ch)
		err := server.Shutdown(context.Background())
		if err != nil {
			fmt.Printf("shutdown error: %s\n", err)
		}
	}
}

func forkProcess() error {
	flags := []string{"-upgrade"}
	fmt.Printf("forkProcess - arg: %v", os.Args[0])
	cmd := exec.Command(os.Args[0], flags...)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	l, _ := ln.(*net.TCPListener)
	lfd, err := l.File()
	if err != nil {
		return err
	}
	cmd.ExtraFiles = []*os.File{lfd}
	cmd.
	return cmd.Start()
}

Reference