Support setting file descriptor limits for the proxy. (GoogleCloudPlatform#179)

Check for file descriptor limit on startup.

Author: easwars

cmd/cloud_sql_proxy/cloud_sql_proxy.go

@@ -38,6 +38,7 @@ import (
 	"github.com/GoogleCloudPlatform/cloudsql-proxy/logging"
 	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/certs"
 	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/fuse"
+	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/limits"
 	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/proxy"
 
 	"cloud.google.com/go/compute/metadata"
@@ -76,6 +77,7 @@ can be removed automatically by this program.`)
 
 	// Settings for limits
 	maxConnections = flag.Uint64("max_connections", 0, `If provided, the maximum number of connections to establish before refusing new connections. Defaults to 0 (no limit)`)
+	fdRlimit       = flag.Uint64("fd_rlimit", limits.ExpectedFDs, `Sets the rlimit on the number of open file descriptors for the proxy to the provided value. If set to zero, disables attempts to set the rlimit. Defaults to a value which can support 4K connections to one instance`)
 
 	// Settings for authentication.
 	token     = flag.String("token", "", "When set, the proxy uses this Bearer token for authorization.")
@@ -410,6 +412,12 @@ func main() {
 		log.SetOutput(ioutil.Discard)
 	}
 
+	if *fdRlimit != 0 {
+		if err := limits.SetupFDLimits(*fdRlimit); err != nil {
+			logging.Infof("failed to setup file descriptor limits: %v", err)
+		}
+	}
+
 	// TODO: needs a better place for consolidation
 	// if instances is blank and env var INSTANCES is supplied use it
 	if envInstances := os.Getenv("INSTANCES"); *instances == "" && envInstances != "" {

proxy/limits/limits.go

@@ -0,0 +1,81 @@
+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+// Package limits provides routines to check and enforce certain resource
+// limits on the Cloud SQL client proxy process.
+package limits
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/GoogleCloudPlatform/cloudsql-proxy/logging"
+)
+
+var (
+	// For overriding in unittests.
+	syscallGetrlimit = syscall.Getrlimit
+	syscallSetrlimit = syscall.Setrlimit
+)
+
+// Each connection handled by the proxy requires two file descriptors, one
+// for the local end of the connection and one for the remote. So, the proxy
+// process should be able to open at least 8K file descriptors if it is to
+// handle 4K connections to one instance.
+const ExpectedFDs = 8500
+
+// SetupFDLimits ensures that the process running the Cloud SQL proxy can have
+// at least wantFDs number of open file descriptors. It returns an error if it
+// cannot ensure the same.
+func SetupFDLimits(wantFDs uint64) error {
+	rlim := &syscall.Rlimit{}
+	if err := syscallGetrlimit(syscall.RLIMIT_NOFILE, rlim); err != nil {
+		return fmt.Errorf("failed to read rlimit for max file descriptors: %v", err)
+	}
+
+	if rlim.Cur >= wantFDs {
+		logging.Verbosef("current FDs rlimit set to %d, wanted limit is %d. Nothing to do here.", rlim.Cur, wantFDs)
+		return nil
+	}
+
+	// Linux man page:
+	// The soft limit is the value that the kernel enforces for the corre‐
+	// sponding resource. The hard limit acts as a ceiling for the soft limit:
+	// an unprivileged process may set only its soft limit to a value in the
+	// range from 0 up to the hard limit, and (irreversibly) lower its hard
+	// limit. A privileged process (under Linux: one with the CAP_SYS_RESOURCE
+	// capability in the initial user namespace) may make arbitrary changes to
+	// either limit value.
+	if rlim.Max < wantFDs {
+		// When the hard limit is less than what is requested, let's just give it a
+		// shot, and if we fail, we fallback and try just setting the softlimit.
+		rlim2 := &syscall.Rlimit{}
+		rlim2.Max = wantFDs
+		rlim2.Cur = wantFDs
+		if err := syscallSetrlimit(syscall.RLIMIT_NOFILE, rlim2); err == nil {
+			logging.Verbosef("Rlimits for file descriptors set to {%v}", rlim2)
+			return nil
+		}
+	}
+
+	rlim.Cur = wantFDs
+	if err := syscallSetrlimit(syscall.RLIMIT_NOFILE, rlim); err != nil {
+		return fmt.Errorf("failed to set rlimit {%v} for max file descriptors: %v", rlim, err)
+	}
+
+	logging.Verbosef("Rlimits for file descriptors set to {%v}", rlim)
+	return nil
+}

proxy/limits/limits_test.go

@@ -0,0 +1,135 @@
+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !windows
+
+package limits
+
+import (
+	"errors"
+	"math"
+	"syscall"
+	"testing"
+)
+
+type rlimitFunc func(int, *syscall.Rlimit) error
+
+func TestSetupFDLimits(t *testing.T) {
+	tests := []struct {
+		desc    string
+		getFunc rlimitFunc
+		setFunc rlimitFunc
+		wantFDs uint64
+		wantErr bool
+	}{
+		{
+			desc: "Getrlimit fails",
+			getFunc: func(_ int, _ *syscall.Rlimit) error {
+				return errors.New("failed to read rlimit for max file descriptors")
+			},
+			setFunc: func(_ int, _ *syscall.Rlimit) error {
+				panic("shouldn't be called")
+			},
+			wantFDs: 0,
+			wantErr: true,
+		},
+		{
+			desc: "Getrlimit max is less than wantFDs",
+			getFunc: func(_ int, rlim *syscall.Rlimit) error {
+				rlim.Cur = 512
+				rlim.Max = 512
+				return nil
+			},
+			setFunc: func(_ int, rlim *syscall.Rlimit) error {
+				if rlim.Cur != 1024 || rlim.Max != 1024 {
+					return errors.New("setrlimit called with unexpected value")
+				}
+				return nil
+			},
+			wantFDs: 1024,
+			wantErr: false,
+		},
+		{
+			desc: "Getrlimit returns rlim_infinity",
+			getFunc: func(_ int, rlim *syscall.Rlimit) error {
+				rlim.Cur = math.MaxUint64
+				rlim.Max = math.MaxUint64
+				return nil
+			},
+			setFunc: func(_ int, _ *syscall.Rlimit) error {
+				panic("shouldn't be called")
+			},
+			wantFDs: 1024,
+			wantErr: false,
+		},
+		{
+			desc: "Getrlimit cur is greater than wantFDs",
+			getFunc: func(_ int, rlim *syscall.Rlimit) error {
+				rlim.Cur = 512
+				rlim.Max = 512
+				return nil
+			},
+			setFunc: func(_ int, _ *syscall.Rlimit) error {
+				panic("shouldn't be called")
+			},
+			wantFDs: 256,
+			wantErr: false,
+		},
+		{
+			desc: "Setrlimit fails",
+			getFunc: func(_ int, rlim *syscall.Rlimit) error {
+				rlim.Cur = 128
+				rlim.Max = 512
+				return nil
+			},
+			setFunc: func(_ int, _ *syscall.Rlimit) error {
+				return errors.New("failed to set rlimit for max file descriptors")
+			},
+			wantFDs: 256,
+			wantErr: true,
+		},
+		{
+			desc: "Success",
+			getFunc: func(_ int, rlim *syscall.Rlimit) error {
+				rlim.Cur = 128
+				rlim.Max = 512
+				return nil
+			},
+			setFunc: func(_ int, _ *syscall.Rlimit) error {
+				return nil
+			},
+			wantFDs: 256,
+			wantErr: false,
+		},
+	}
+
+	for _, test := range tests {
+		oldGetFunc := syscallGetrlimit
+		syscallGetrlimit = test.getFunc
+		defer func() {
+			syscallGetrlimit = oldGetFunc
+		}()
+
+		oldSetFunc := syscallSetrlimit
+		syscallSetrlimit = test.setFunc
+		defer func() {
+			syscallSetrlimit = oldSetFunc
+		}()
+
+		gotErr := SetupFDLimits(test.wantFDs)
+		if (gotErr != nil) != test.wantErr {
+			t.Errorf("%s: limits.SetupFDLimits(%d) returned error %v, wantErr %v", test.desc, test.wantFDs, gotErr, test.wantErr)
+		}
+	}
+}

proxy/limits/limits_windows.go

@@ -0,0 +1,30 @@
+// Copyright 2015 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package limits is a package stub for windows, and we currently don't support
+// setting limits in windows.
+package limits
+
+import "errors"
+
+// We don't support limit on the number of file handles in windows.
+const ExpectedFDs = 0
+
+func SetupFDLimits(wantFDs uint64) error {
+	if wantFDs != 0 {
+		return errors.New("setting limits on the number of file handles is not supported")
+	}
+
+	return nil
+}