From 12b67cad9296064f892360395752cef81684bccd Mon Sep 17 00:00:00 2001 From: tombogle Date: Thu, 26 Feb 2026 09:12:34 -0500 Subject: [PATCH 1/3] Modified RobustNetworkOperation to send a UserAgent string to try to prevent CI test failure. --- SIL.Core.Desktop/Network/RobustNetworkOperation.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/SIL.Core.Desktop/Network/RobustNetworkOperation.cs b/SIL.Core.Desktop/Network/RobustNetworkOperation.cs index 960813224..fe906ba88 100644 --- a/SIL.Core.Desktop/Network/RobustNetworkOperation.cs +++ b/SIL.Core.Desktop/Network/RobustNetworkOperation.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Net; using System.Security.Cryptography; using System.Text; @@ -8,6 +8,9 @@ namespace SIL.Network public class RobustNetworkOperation { + private const string DefaultUserAgent = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) libpalaso"; + /// /// Perform a web action, trying various things to use a proxy if needed, including requesting /// (and remembering) user credentials from the user. @@ -109,7 +112,7 @@ public static string GetClearText(string encryptedString) } /// - /// Used by Chorus to get proxy name, user name, and password of the remote repository. + /// Used by Chorus to get proxy name, username, and password of the remote repository. /// /// true if a proxy is needed. THROWS if it just can't get through public static bool DoHttpGetAndGetProxyInfo(string url, out string hostAndPort, out string userName, out string password, Action verboseLog) @@ -125,6 +128,7 @@ public static bool DoHttpGetAndGetProxyInfo(string url, out string hostAndPort, proxy => { client.Proxy = proxy; + client.Headers[HttpRequestHeader.UserAgent] = DefaultUserAgent; client.DownloadData(url); //we don't actually care what comes back }, verboseLog From ab62111a50c95b72e0a3ef4aa1c93ab4b860338e Mon Sep 17 00:00:00 2001 From: tombogle Date: Thu, 26 Feb 2026 17:05:45 -0500 Subject: [PATCH 2/3] Added comments about the need for the user agent string --- SIL.Core.Desktop/Network/RobustNetworkOperation.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/SIL.Core.Desktop/Network/RobustNetworkOperation.cs b/SIL.Core.Desktop/Network/RobustNetworkOperation.cs index fe906ba88..8e12560c5 100644 --- a/SIL.Core.Desktop/Network/RobustNetworkOperation.cs +++ b/SIL.Core.Desktop/Network/RobustNetworkOperation.cs @@ -8,6 +8,8 @@ namespace SIL.Network public class RobustNetworkOperation { + // This user agent string seems to be satisfactory for convincing the servers that we are + // a real browser, and not some sort of bot. private const string DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) libpalaso"; @@ -128,6 +130,10 @@ public static bool DoHttpGetAndGetProxyInfo(string url, out string hostAndPort, proxy => { client.Proxy = proxy; + // The following convinces sites that this request is coming from a real + // browser, and not some sort of bot. Seems to be necessary now at least when + // running tests in CI, and it might also be helpful in other circumstances to + // avoid getting blocked by some sites. client.Headers[HttpRequestHeader.UserAgent] = DefaultUserAgent; client.DownloadData(url); //we don't actually care what comes back From fe2a8e8bee5fe303423b6c9ea7fdccacc6cc85b6 Mon Sep 17 00:00:00 2001 From: tombogle Date: Thu, 26 Feb 2026 17:35:32 -0500 Subject: [PATCH 3/3] Added an optional userAgentHeader parameter to DoHttpGetAndGetProxyInfo to allow a client to mimic a real browser if necessary. Also improved the comments --- CHANGELOG.md | 2 + .../Network/RobustNetworkOperation.Tests.cs | 5 +- .../Network/RobustNetworkOperation.cs | 69 +++++++++++++++---- 3 files changed, 59 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d849439b3..d5356ac4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [SIL.Core.Clearshare] New methods "GetIsStringAvailableForLangId" and "GetDynamicStringOrEnglish" were added to Localizer for use in LicenseInfo's "GetBestLicenseTranslation" method, to remove LicenseInfo's L10NSharp dependency. - [SIL.Windows.Forms.Clearshare] New ILicenseWithImage interface handles "GetImage" method for Winforms-dependent licenses, implemented in CreativeCommonsLicense and CustomLicense, and formerly included in LicenseInfo. - [SIL.Core.Clearshare] New tests MetadataBareTests are based on previous MetadataTests in SIL.Windows.Forms.Clearshare. The tests were updated to use ImageSharp instead of Winforms for handling images. +- [SIL.Core.Desktop] Added a constant (kBrowserCompatibleUserAgent) to RobustNetworkOperation: a browser-like User Agent string that can be used when making HTTP requests to strict servers. ### Fixed - [SIL.DictionaryServices] Fix memory leak in LiftWriter @@ -49,6 +50,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - [SIL.Windows.Forms] BREAKING CHANGE: ToolStripExtensions.InitializeWithAvailableUILocales() removed the ILocalizationManager parameter. This method no longer provides functionality to display the localization dialog box in response to the user clicking More. - [SIL.Windows.Forms] BREAKING CHANGE: Removed optional moreSelected parameter from ToolStripExtensions.InitializeWithAvailableUILocales method. This parameter was no longer being used. Clients that want to have a More menu item that performs a custom action will now need to add it themselves. - [SIL.Windows.Forms] BREAKING CHANGE: LocalizationIncompleteDlg's EmailAddressForLocalizationRequests is no longer autopopulated from LocalizationManager. A new optional constructor parameter, emailAddressForLocalizationRequests, can be used instead. If not supplied, the "More information" controls will be hidden. +- [SIL.Core.Desktop] Added an optional userAgentHeader parameter to DoHttpGetAndGetProxyInfo to allow a client to mimic a real browser if necessary. ### Removed - [SIL.Windows.Forms] In .NET 8 builds, removed Scanner and Camera options from the Image Toolbox. diff --git a/SIL.Core.Desktop.Tests/Network/RobustNetworkOperation.Tests.cs b/SIL.Core.Desktop.Tests/Network/RobustNetworkOperation.Tests.cs index 0e23ea73a..d78a542dc 100644 --- a/SIL.Core.Desktop.Tests/Network/RobustNetworkOperation.Tests.cs +++ b/SIL.Core.Desktop.Tests/Network/RobustNetworkOperation.Tests.cs @@ -1,4 +1,4 @@ -using System.Diagnostics; +using System.Diagnostics; using NUnit.Framework; using SIL.Network; @@ -19,7 +19,8 @@ public void DoHttpGetAndGetProxyInfo_404() public void DoHttpGetAndGetProxyInfo_NoProxy_ReturnsFalse() { var gotProxy = RobustNetworkOperation.DoHttpGetAndGetProxyInfo( - "https://sil.org/", out _, out _, out _, s => Debug.WriteLine(s)); + "https://sil.org/", out _, out _, out _, s => Debug.WriteLine(s), + RobustNetworkOperation.kBrowserCompatibleUserAgent); Assert.That(gotProxy, Is.False); } } diff --git a/SIL.Core.Desktop/Network/RobustNetworkOperation.cs b/SIL.Core.Desktop/Network/RobustNetworkOperation.cs index 8e12560c5..2cc8173a2 100644 --- a/SIL.Core.Desktop/Network/RobustNetworkOperation.cs +++ b/SIL.Core.Desktop/Network/RobustNetworkOperation.cs @@ -8,9 +8,12 @@ namespace SIL.Network public class RobustNetworkOperation { - // This user agent string seems to be satisfactory for convincing the servers that we are - // a real browser, and not some sort of bot. - private const string DefaultUserAgent = + /// + /// A browser-like User Agent string that can be used when making HTTP requests to servers + /// that reject requests lacking a typical browser User-Agent header. Used in tests and + /// available to callers that encounter 403 responses due to restrictive server filtering. + /// + public const string kBrowserCompatibleUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) libpalaso"; /// @@ -27,7 +30,7 @@ public static IWebProxy Do(Action action, Action verboseLog) var proxy = new WebProxy(); action(proxy); - //!!!!!!!!!!!!!! in Sept 2011, hatton disabled proxy lookup. It was reportedly causing grief in Nigeria, + //!!!!!!!!!!!!!! in Sept 2011, Hatton disabled proxy lookup. It was reportedly causing grief in Nigeria, //asking for credentials over and over, and SIL PNG doesn't use a proxy anymore. So for now... @@ -114,10 +117,51 @@ public static string GetClearText(string encryptedString) } /// - /// Used by Chorus to get proxy name, username, and password of the remote repository. + /// Used to determine whether an HTTP GET request requires a proxy and, if so, to retrieve + /// the proxy host and credentials needed to access the specified remote repository URL. /// - /// true if a proxy is needed. THROWS if it just can't get through - public static bool DoHttpGetAndGetProxyInfo(string url, out string hostAndPort, out string userName, out string password, Action verboseLog) + /// + /// The full URL to send an HTTP GET request to. Used to test connectivity and determine + /// whether proxy credentials are required. + /// + /// + /// Outputs the proxy host (including port, if applicable) if a proxy is required; + /// otherwise, an empty string. + /// + /// + /// Outputs the proxy username if authentication is required; otherwise an empty string. + /// + /// + /// Outputs the proxy password if authentication is required; otherwise an empty string. + /// + /// + /// Optional callback for logging diagnostic information about the request and proxy + /// detection process. + /// + /// + /// Optional user agent header string to send with the request. Some servers require a + /// browser-like user agent and may reject requests that appear to come from automated + /// clients. See for an example value. + /// + /// + /// True if a proxy is required and credentials were obtained; false if no proxy is needed. + /// + /// + /// Thrown if is null. + /// + /// + /// Thrown if is not a valid URI. + /// + /// + /// Thrown if the HTTP request fails due to network errors, proxy configuration + /// issues, or authentication failures. + /// + /// + /// Thrown if the URI scheme is not supported. + /// + public static bool DoHttpGetAndGetProxyInfo(string url, out string hostAndPort, + out string userName, out string password, Action verboseLog, + string userAgentHeader = null) { hostAndPort = string.Empty; userName = string.Empty; @@ -130,11 +174,8 @@ public static bool DoHttpGetAndGetProxyInfo(string url, out string hostAndPort, proxy => { client.Proxy = proxy; - // The following convinces sites that this request is coming from a real - // browser, and not some sort of bot. Seems to be necessary now at least when - // running tests in CI, and it might also be helpful in other circumstances to - // avoid getting blocked by some sites. - client.Headers[HttpRequestHeader.UserAgent] = DefaultUserAgent; + if (userAgentHeader != null) + client.Headers[HttpRequestHeader.UserAgent] = userAgentHeader; client.DownloadData(url); //we don't actually care what comes back }, verboseLog @@ -160,9 +201,7 @@ public static bool DoHttpGetAndGetProxyInfo(string url, out string hostAndPort, var networkCredential = proxyInfo.Credentials.GetCredential(destination, ""); userName = networkCredential.UserName; password = networkCredential.Password; - if (verboseLog != null) - verboseLog.Invoke("DoHttpGetAndGetProxyInfo Returning with credentials. UserName is " + userName); - + verboseLog?.Invoke("DoHttpGetAndGetProxyInfo Returning with credentials. UserName is " + userName); return true; }