Added configuration option for proxy (#609)

* Use downloader.SetDefaultConfig() to set user-agent for the arduino-cli

This change allows to not pass-trough the downloader configuration from
function to function everywhere (and sometime we forget to pass it for
example in the "core update-index" command).

* Add support for network proxy configuration

* Cosmetic: added blank space in configuration/defaults.go

* Update go-downloader to 1.2.0

* Refactored function to generate user-agent

* Added a TODO for missing proxy usage in board list API query

* Partially revert global network configuration.

It's better to read the network configuration before each http request so
in case it is changed using "Settings" functions in daemon mode the changes
are immediately applied.

Author: cmaglie

arduino/cores/packagemanager/download.go

@@ -17,7 +17,6 @@ package packagemanager
 
 import (
 	"fmt"
-	"net/http"
 
 	"github.com/arduino/arduino-cli/arduino/cores"
 	"go.bug.st/downloader"
@@ -101,16 +100,16 @@ func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReferenc
 
 // DownloadToolRelease downloads a ToolRelease. If the tool is already downloaded a nil Downloader
 // is returned.
-func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, downloaderHeaders http.Header) (*downloader.Downloader, error) {
+func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *downloader.Config) (*downloader.Downloader, error) {
 	resource := tool.GetCompatibleFlavour()
 	if resource == nil {
 		return nil, fmt.Errorf("tool not available for your OS")
 	}
-	return resource.Download(pm.DownloadDir, downloaderHeaders)
+	return resource.Download(pm.DownloadDir, config)
 }
 
 // DownloadPlatformRelease downloads a PlatformRelease. If the platform is already downloaded a
 // nil Downloader is returned.
-func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, downloaderHeaders http.Header) (*downloader.Downloader, error) {
-	return platform.Resource.Download(pm.DownloadDir, downloaderHeaders)
+func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config) (*downloader.Downloader, error) {
+	return platform.Resource.Download(pm.DownloadDir, config)
 }

arduino/libraries/librariesmanager/download.go

@@ -25,8 +25,8 @@ import (
 var LibraryIndexURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json")
 
 // UpdateIndex downloads the libraries index file from Arduino repository.
-func (lm *LibrariesManager) UpdateIndex() (*downloader.Downloader, error) {
+func (lm *LibrariesManager) UpdateIndex(config *downloader.Config) (*downloader.Downloader, error) {
 	lm.IndexFile.Parent().MkdirAll()
 	// TODO: Download from gzipped URL index
-	return downloader.Download(lm.IndexFile.String(), LibraryIndexURL.String(), downloader.NoResume)
+	return downloader.DownloadWithConfig(lm.IndexFile.String(), LibraryIndexURL.String(), *config, downloader.NoResume)
 }

arduino/resources/helpers.go

@@ -17,7 +17,6 @@ package resources
 
 import (
 	"fmt"
-	"net/http"
 	"os"
 
 	"github.com/arduino/go-paths-helper"
@@ -44,7 +43,7 @@ func (r *DownloadResource) IsCached(downloadDir *paths.Path) (bool, error) {
 }
 
 // Download a DownloadResource.
-func (r *DownloadResource) Download(downloadDir *paths.Path, downloaderHeaders http.Header) (*downloader.Downloader, error) {
+func (r *DownloadResource) Download(downloadDir *paths.Path, config *downloader.Config) (*downloader.Downloader, error) {
 	cached, err := r.TestLocalArchiveIntegrity(downloadDir)
 	if err != nil {
 		return nil, fmt.Errorf("testing local archive integrity: %s", err)
@@ -72,7 +71,5 @@ func (r *DownloadResource) Download(downloadDir *paths.Path, downloaderHeaders h
 		return nil, fmt.Errorf("getting archive file info: %s", err)
 	}
 
-	downloadConfig := downloader.Config{
-		RequestHeaders: downloaderHeaders}
-	return downloader.DownloadWithConfig(path.String(), r.URL, downloadConfig)
+	return downloader.DownloadWithConfig(path.String(), r.URL, *config)
 }

arduino/resources/helpers_test.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
+	"go.bug.st/downloader"
 )
 
 type EchoHandler struct{}
@@ -52,7 +53,7 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) {
 		URL:             srv.URL,
 	}
 
-	d, err := r.Download(tmp, http.Header{"User-Agent": []string{goldUserAgentValue}})
+	d, err := r.Download(tmp, &downloader.Config{RequestHeaders: http.Header{"User-Agent": []string{goldUserAgentValue}}})
 	require.NoError(t, err)
 	err = d.Run()
 	require.NoError(t, err)

arduino/resources/resources_test.go

@@ -18,11 +18,11 @@ package resources
 import (
 	"crypto"
 	"encoding/hex"
-	"net/http"
 	"testing"
 
 	"github.com/arduino/go-paths-helper"
 	"github.com/stretchr/testify/require"
+	"go.bug.st/downloader"
 )
 
 func TestDownloadAndChecksums(t *testing.T) {
@@ -42,7 +42,7 @@ func TestDownloadAndChecksums(t *testing.T) {
 	require.NoError(t, err)
 
 	downloadAndTestChecksum := func() {
-		d, err := r.Download(tmp, http.Header{})
+		d, err := r.Download(tmp, &downloader.Config{})
 		require.NoError(t, err)
 		err = d.Run()
 		require.NoError(t, err)
@@ -58,7 +58,7 @@ func TestDownloadAndChecksums(t *testing.T) {
 	downloadAndTestChecksum()
 
 	// Download with cached file
-	d, err := r.Download(tmp, http.Header{})
+	d, err := r.Download(tmp, &downloader.Config{})
 	require.NoError(t, err)
 	require.Nil(t, d)
 

cli/core/download.go

@@ -66,8 +66,7 @@ func runDownloadCommand(cmd *cobra.Command, args []string) {
 			Architecture:    platformRef.Architecture,
 			Version:         platformRef.Version,
 		}
-		_, err := core.PlatformDownload(context.Background(), platformDownloadreq, output.ProgressBar(),
-			globals.NewHTTPClientHeader())
+		_, err := core.PlatformDownload(context.Background(), platformDownloadreq, output.ProgressBar())
 		if err != nil {
 			feedback.Errorf("Error downloading %s: %v", args[i], err)
 			os.Exit(errorcodes.ErrNetwork)

cli/core/install.go

@@ -67,8 +67,7 @@ func runInstallCommand(cmd *cobra.Command, args []string) {
 			Architecture:    platformRef.Architecture,
 			Version:         platformRef.Version,
 		}
-		_, err := core.PlatformInstall(context.Background(), plattformInstallReq, output.ProgressBar(),
-			output.TaskProgress(), globals.NewHTTPClientHeader())
+		_, err := core.PlatformInstall(context.Background(), plattformInstallReq, output.ProgressBar(), output.TaskProgress())
 		if err != nil {
 			feedback.Errorf("Error during install: %v", err)
 			os.Exit(errorcodes.ErrGeneric)

cli/core/upgrade.go

@@ -93,7 +93,7 @@ func runUpgradeCommand(cmd *cobra.Command, args []string) {
 			Architecture:    platformRef.Architecture,
 		}
 
-		_, err := core.PlatformUpgrade(context.Background(), r, output.ProgressBar(), output.TaskProgress(), globals.NewHTTPClientHeader())
+		_, err := core.PlatformUpgrade(context.Background(), r, output.ProgressBar(), output.TaskProgress())
 		if err == core.ErrAlreadyLatest {
 			feedback.Printf("Platform %s is already at the latest version", platformRef)
 		} else if err != nil {

cli/daemon/daemon.go

@@ -21,9 +21,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net"
-	"net/http"
 	"os"
-	"runtime"
 	"syscall"
 
 	"github.com/arduino/arduino-cli/cli/errorcodes"
@@ -67,26 +65,15 @@ func runDaemonCommand(cmd *cobra.Command, args []string) {
 		stats.Incr("daemon", stats.T("success", "true"))
 		defer stats.Flush()
 	}
-
 	port := viper.GetString("daemon.port")
 	s := grpc.NewServer()
 
-	// Compose user agent header
-	headers := http.Header{
-		"User-Agent": []string{
-			fmt.Sprintf("%s/%s daemon (%s; %s; %s) Commit:%s",
-				globals.VersionInfo.Application,
-				globals.VersionInfo.VersionString,
-				runtime.GOARCH,
-				runtime.GOOS,
-				runtime.Version(),
-				globals.VersionInfo.Commit),
-		},
-	}
-	// Register the commands service
+	// Set specific user-agent for the daemon
+	viper.Set("network.user_agent_ext", "daemon")
+
+	// register the commands service
 	srv_commands.RegisterArduinoCoreServer(s, &daemon.ArduinoCoreServerImpl{
-		DownloaderHeaders: headers,
-		VersionString:     globals.VersionInfo.VersionString,
+		VersionString: globals.VersionInfo.VersionString,
 	})
 
 	// Register the monitors service

cli/globals/globals.go

@@ -33,8 +33,16 @@ var (
 )
 
 // NewHTTPClientHeader returns the http.Header object that must be used by the clients inside the downloaders
-func NewHTTPClientHeader() http.Header {
-	userAgentValue := fmt.Sprintf("%s/%s (%s; %s; %s) Commit:%s", VersionInfo.Application,
-		VersionInfo.VersionString, runtime.GOARCH, runtime.GOOS, runtime.Version(), VersionInfo.Commit)
+// and adds the subComponent if specified
+func NewHTTPClientHeader(subComponent string) http.Header {
+	if subComponent != "" {
+		subComponent = " " + subComponent
+	}
+	userAgentValue := fmt.Sprintf("%s/%s%s (%s; %s; %s) Commit:%s",
+		VersionInfo.Application,
+		VersionInfo.VersionString,
+		subComponent,
+		runtime.GOARCH, runtime.GOOS, runtime.Version(),
+		VersionInfo.Commit)
 	return http.Header{"User-Agent": []string{userAgentValue}}
 }