Graceful restart

[kennygrant]

Add graceful restart, responding to SIGHUP (perhaps via a callback to app?)

[abishekmuthian]

I have implemented graceful restart using signals,

type listener struct {
	Addr     string `json:"addr"`
	FD       int    `json:"fd"`
	Filename string `json:"filename"`
}

func importListener(addr string) (net.Listener, error) {
	// Extract the encoded listener metadata from the environment.
	listenerEnv := os.Getenv("LISTENER")
	if listenerEnv == "" {
		return nil, fmt.Errorf("unable to find LISTENER environment variable")
	}

	// Unmarshal the listener metadata.
	var l listener
	err := json.Unmarshal([]byte(listenerEnv), &l)
	if err != nil {
		return nil, err
	}
	if l.Addr != addr {
		return nil, fmt.Errorf("unable to find listener for %v", addr)
	}

	// The file has already been passed to this process, extract the file
	// descriptor and name from the metadata to rebuild/find the *os.File for
	// the listener.
	listenerFile := os.NewFile(uintptr(l.FD), l.Filename)
	if listenerFile == nil {
		return nil, fmt.Errorf("unable to create listener file: %v", err)
	}
	defer listenerFile.Close()

	// Create a net.Listener from the *os.File.
	ln, err := net.FileListener(listenerFile)
	if err != nil {
		return nil, err
	}

	return ln, nil
}

func createListener(addr string) (net.Listener, error) {
	ln, err := net.Listen("tcp", addr)
	if err != nil {
		return nil, err
	}

	return ln, nil
}

func createOrImportListener(addr string) (net.Listener, error) {
	// Try and import a listener for addr. If it's found, use it.
	ln, err := importListener(addr)
	if err == nil {
		log.Info(log.V{"msg": "Imported listener file descriptor for", "Address": addr})
		return ln, nil
	}

	// No listener was imported, that means this process has to create one.
	ln, err = createListener(addr)
	if err != nil {
		return nil, err
	}

	log.Info(log.V{"msg": "Created listener file descriptor for", "Address": addr})

	return ln, nil
}

func getListenerFile(ln net.Listener) (*os.File, error) {
	switch t := ln.(type) {
	case *net.TCPListener:
		return t.File()
	case *net.UnixListener:
		return t.File()
	}
	return nil, fmt.Errorf("unsupported listener: %T", ln)
}

func forkChild(addr string, ln net.Listener) (*os.Process, error) {
	// Get the file descriptor for the listener and marshal the metadata to pass
	// to the child in the environment.
	lnFile, err := getListenerFile(ln)
	if err != nil {
		return nil, err
	}
	defer lnFile.Close()
	l := listener{
		Addr:     addr,
		FD:       3,
		Filename: lnFile.Name(),
	}
	listenerEnv, err := json.Marshal(l)
	if err != nil {
		return nil, err
	}

	// Pass stdin, stdout, and stderr along with the listener to the child.
	files := []*os.File{
		os.Stdin,
		os.Stdout,
		os.Stderr,
		lnFile,
	}

	// Get current environment and add in the listener to it.
	environment := append(os.Environ(), "LISTENER="+string(listenerEnv))

	// Get current process name and directory.
	execName, err := os.Executable()
	if err != nil {
		return nil, err
	}
	execDir := filepath.Dir(execName)

	// Spawn child process.
	p, err := os.StartProcess(execName, []string{execName}, &os.ProcAttr{
		Dir:   execDir,
		Env:   environment,
		Files: files,
		Sys:   &syscall.SysProcAttr{},
	})
	if err != nil {
		return nil, err
	}

	return p, nil
}

func waitForSignals(addr string, ln net.Listener, server *http.Server) error {
	signalCh := make(chan os.Signal, 1024)
	signal.Notify(signalCh, syscall.SIGHUP, syscall.SIGUSR2, syscall.SIGINT, syscall.SIGQUIT)
	for {
		select {
		case s := <-signalCh:
			log.Info(log.V{"msg": "Received Signal", "Signal": s})
			switch s {
			case syscall.SIGHUP:
				// Fork a child process.
				p, err := forkChild(addr, ln)
				if err != nil {
					log.Error(log.V{"msg": "Unable to fork child", "Error": err})
					continue
				}
				log.Info(log.V{"msg": "Forked child", "Pid": p.Pid})

				// Create a context that will expire in 5 seconds and use this as a
				// timeout to Shutdown.
				ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
				defer cancel()

				// Return any errors during shutdown.
				return server.Shutdown(ctx)
			case syscall.SIGUSR2:
				// Fork a child process.
				p, err := forkChild(addr, ln)
				if err != nil {
					log.Error(log.V{"msg": "Unable to fork child", "Error": err})
					continue
				}

				// Print the PID of the forked process and keep waiting for more signals.
				log.Info(log.V{"msg": "Forked child", "Pid": p.Pid})
			case syscall.SIGINT, syscall.SIGQUIT:
				// Create a context that will expire in 5 seconds and use this as a
				// timeout to Shutdown.
				ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
				defer cancel()

				// Return any errors during shutdown.
				return server.Shutdown(ctx)
			}
		}
	}
}

// watchProcess initialises OS signal and fork process
func watchProcess(server *http.Server, ln net.Listener) {
	go func() {
		time.Sleep(5 * time.Second)
		err := waitForSignals(server.Addr, ln, server)
		if err != nil {
			log.Error(log.V{"msg": "Exiting", "Error": err})
			return
		}
	}()
}

// Start starts an http server on the given port
func (s *Server) Start() error {
	server := &http.Server{
		// Set the port in the preferred string format
		Addr: s.PortString(),

		// The default server from net/http has no timeouts - set some limits
		ReadHeaderTimeout: 30 * time.Second,
		ReadTimeout:       60 * time.Second,
		WriteTimeout:      60 * time.Second,
		IdleTimeout:       10 * time.Second, // IdleTimeout was introduced in Go 1.8

	}
	ln, err := createOrImportListener(server.Addr)
	if err != nil {
		log.Error(log.V{"msg": "Unable to create or import a listener", "Error": err})
		os.Exit(1)
	}
	if err == nil {
		watchProcess(server, ln)
	}

	return server.Serve(ln)
}

// StartTLSAuto starts an https server on the given port
// by requesting certs from an ACME provider using the http-01 challenge.
// it also starts a server on the port 80 to listen for challenges and redirect
// The server must be on a public IP which matches the
// DNS for the domains.
func (s *Server) StartTLSAuto(email, domains string) error {
	autocertDomains := strings.Split(domains, " ")
	certManager := &autocert.Manager{
		Prompt:     autocert.AcceptTOS,
		Email:      email,                                      // Email for problems with certs
		HostPolicy: autocert.HostWhitelist(autocertDomains...), // Domains to request certs for
		Cache:      autocert.DirCache("certs"),                 // Cache certs in secrets folder
	}
	// Handle all :80 traffic using autocert to allow http-01 challenge responses
	go func() {
		http.ListenAndServe(":80", certManager.HTTPHandler(nil))
	}()

	server := s.ConfiguredTLSServer(certManager)
	ln, err := createOrImportListener(server.Addr)
	if err != nil {
		log.Error(log.V{"msg": "Unable to create or import a listener", "Error": err})
		os.Exit(1)
	}
	if err == nil {
		watchProcess(server, ln)
	}

	return server.ServeTLS(ln, "", "")
}