[net] Add libcurl based support for files on HTTP remotes

.github/workflows/root-ci-config/buildconfig/alma8.txt

@@ -2,5 +2,6 @@ builtin_gtest=ON
 builtin_nlohmannjson=ON
 builtin_tbb=ON
 builtin_vdt=ON
+curl=OFF
 pythia8=ON
 tmva-sofie=ON

.github/workflows/root-ci-config/buildconfig/global.txt

@@ -37,6 +37,7 @@ clingtest=OFF
 cocoa=OFF
 coverage=OFF
 cuda=OFF
+curl=ON
 daos=OFF
 dataframe=ON
 davix=ON

cmake/modules/RootBuildOptions.cmake

@@ -120,6 +120,7 @@ ROOT_BUILD_OPTION(clad ON "Build clad, the cling automatic differentiation plugi
 ROOT_BUILD_OPTION(cocoa OFF "Use native Cocoa/Quartz graphics backend (MacOS X only)")
 ROOT_BUILD_OPTION(coverage OFF "Enable compile flags for coverage testing")
 ROOT_BUILD_OPTION(cuda OFF "Enable support for CUDA (requires CUDA toolkit >= 7.5)")
+ROOT_BUILD_OPTION(curl OFF "Enable support for HTTP(S) through libcurl")
 ROOT_BUILD_OPTION(daos OFF "Enable RNTuple support for Intel DAOS")
 ROOT_BUILD_OPTION(dataframe ON "Enable ROOT RDataFrame")
 ROOT_BUILD_OPTION(davix ON "Enable support for Davix (HTTP/WebDAV access)")
@@ -220,6 +221,7 @@ if(all)
  set(asimage_tiff_defvalue ON)
  set(cefweb_defvalue ON)
  set(clad_defvalue ON)
+ set(curl_defvalue ON)
  set(dataframe_defvalue ON)
  set(davix_defvalue ON)
  set(dcache_defvalue ON)

cmake/modules/RootConfiguration.cmake

@@ -169,6 +169,16 @@ endif()
 
 set(buildnetxng ${value${netxng}})
 
+set(buildcurl ${value${curl}})
+set(curllibdir ${CURL_LIBRARY_DIR})
+set(curllib ${CURL_LIBRARY})
+set(curlincdir ${CURL_INCLUDE_DIR})
+if(curl)
+  set(hascurl define)
+else()
+  set(hascurl undef)
+endif()
+
 set(builddcap ${value${dcap}})
 set(dcaplibdir ${DCAP_LIBRARY_DIR})
 set(dcaplib ${DCAP_LIBRARY})

cmake/modules/SearchInstalledSoftware.cmake

@@ -1192,6 +1192,32 @@ if(builtin_davix)
   endif()
 endif()
 
+#---Check for curl library-----------------------------------------------------------
+foreach(suffix FOUND INCLUDE_DIR INCLUDE_DIRS LIBRARY LIBRARIES)
+  unset(CURL_${suffix} CACHE)
+endforeach()
+
+if(curl)
+  message(STATUS "Looking for libcurl")
+  if(MSVC)
+    # On Windows, we must initialize libcurl lazily (not in a static [DLL] initializer), and this
+    # works only safely as of 7.84 with the threadsafe option
+    find_package(CURL 7.84 COMPONENTS HTTP HTTPS threadsafe)
+  else()
+    # Matches the libcurl version on EL9/10
+    find_package(CURL 7.76 COMPONENTS HTTP HTTPS)
+  endif()
+
+  if(NOT CURL_FOUND)
+    if(fail-on-missing)
+      message(SEND_ERROR "libcurl not found and curl option required")
+    else()
+      message(STATUS "libcurl not found. Switching off curl option")
+      set(curl OFF CACHE BOOL "Disabled because libcurl was not found (${curl_description})" FORCE)
+    endif()
+  endif()
+endif()
+
 #---Check for liburing----------------------------------------------------------------
 if (uring)
   if(NOT CMAKE_SYSTEM_NAME MATCHES Linux)

config/RConfigure.in

@@ -47,6 +47,7 @@
 #@hascefweb@ R__HAS_CEFWEB  /**/
 #@hasqt6webengine@ R__HAS_QT6WEB  /**/
 #@hasdavix@ R__HAS_DAVIX  /**/
+#@hascurl@ R__HAS_CURL  /**/
 #@hasdataframe@ R__HAS_DATAFRAME /**/
 #@hasroot7@ R__HAS_ROOT7 /**/
 #@use_less_includes@ R__LESS_INCLUDES /**/

config/rootrc.in

@@ -593,6 +593,9 @@ Davix.UseOldClient: @useoldwebfile@
 # S3_ACCESS_KEY, S3_SECRET_KEY, S3_REGION, S3_TOKEN
 # The gEnv vars have higher priority over the these envvars.
 
+# If set to yes, use the curl TFile and RRawFile backend even if davix is available
+Curl.ReplaceDavix: no
+
 # Example of custom setting for the Rint application (root.exe).
 # This overrides the default specified above for a generic application.
 # Color 5 is yellow.

etc/plugins/ROOT@@Internal@@RRawFile/P015_RRawFileCurl.C

@@ -0,0 +1,11 @@
+void P015_RRawFileCurl()
+{
+   TString configfeatures = gROOT->GetConfigFeatures();
+
+   if (configfeatures.Contains("curl") &&
+       (!configfeatures.Contains("davix") || gEnv->GetValue("Curl.ReplaceDavix", 0))) {
+
+      gPluginMgr->AddHandler("ROOT::Internal::RRawFile", "^http[s]?:", "ROOT::Internal::RRawFileCurl", "RCurlHttp",
+                             "RRawFileCurl(std::string_view, ROOT::Internal::RRawFile::ROptions)");
+   }
+}

etc/plugins/TFile/P140_TCurlFile.C

@@ -0,0 +1,10 @@
+void P140_TCurlFile()
+{
+   TString configfeatures = gROOT->GetConfigFeatures();
+
+   if (configfeatures.Contains("curl") &&
+       (!configfeatures.Contains("davix") || gEnv->GetValue("Curl.ReplaceDavix", 0))) {
+
+      gPluginMgr->AddHandler("TFile", "^http[s]?:", "TCurlFile", "RCurlHttp", "TCurlFile(const char *, Option_t *)");
+   }
+}

io/io/src/RRawFile.cxx

@@ -71,12 +71,10 @@ ROOT::Internal::RRawFile::Create(std::string_view url, ROptions options)
       return std::unique_ptr<RRawFile>(new RRawFileUnix(url, options));
 #endif
    }
-   if (transport == "http" || transport == "https" ||
-       transport == "root" || transport == "roots" ) {
-      std::string plgclass = transport.compare( 0, 4, "http" ) == 0 ?
-                             "RRawFileDavix" : "RRawFileNetXNG";
-      if (TPluginHandler *h = gROOT->GetPluginManager()->
-          FindHandler("ROOT::Internal::RRawFile", std::string(url).c_str())) {
+   if (transport == "http" || transport == "https" || transport == "root" || transport == "roots") {
+      std::string plgclass = transport.compare(0, 4, "http") == 0 ? "RRawFileDavix" : "RRawFileNetXNG";
+      if (TPluginHandler *h =
+             gROOT->GetPluginManager()->FindHandler("ROOT::Internal::RRawFile", std::string(url).c_str())) {
          if (h->LoadPlugin() == 0) {
             return std::unique_ptr<RRawFile>(reinterpret_cast<RRawFile *>(h->ExecPlugin(2, &url, &options)));
          }

io/io/test/RRawFile.cxx

@@ -99,7 +99,7 @@ TEST(RRawFile, Basic)
 
 TEST(RRawFile, Remote)
 {
-#ifdef R__HAS_DAVIX
+#if defined(R__HAS_DAVIX) || defined(R__HAS_CURL)
    auto f = RRawFile::Create("http://root.cern/files/davix.test");
    std::string line;
    EXPECT_TRUE(f->Readln(line));

io/io/test/TFileTests.cxx

@@ -131,9 +131,7 @@ void TestReadWithoutGlobalRegistrationIfPossible(const char *fname)
 }
 
 // https://github.com/root-project/root/issues/10742
-#ifndef R__WIN32
-// We prefer not to read remotely files from Windows, if possible
-#ifdef R__HAS_DAVIX
+#if defined(R__HAS_DAVIX) || defined(R__HAS_CURL)
 TEST(TFile, ReadWithoutGlobalRegistrationWeb)
 {
    const auto webFile = "http://root.cern/files/h1/dstarmb.root";
@@ -154,7 +152,6 @@ TEST(TFile, ReadWithCacheWithoutGlobalRegistration)
    gSystem->Unlink("./files");
 }
 #endif
-#endif
 
 // https://github.com/root-project/root/issues/16189
 TEST(TFile, k630forwardCompatibility)

io/io/test/rfile.cxx

@@ -77,7 +77,7 @@ TEST(RFile, OpenInexistent)
    ROOT::TestSupport::CheckDiagsRAII diags;
    diags.optionalDiag(kSysError, "TFile::TFile", "", false);
    diags.optionalDiag(kError, "TFile::TFile", "", false);
-   
+
    try {
       auto f = RFile::Open("does_not_exist.root");
       FAIL() << "trying to open an inexistent file should throw";
@@ -508,7 +508,7 @@ TEST(RFile, IterateKeysOnlyDirsNonRecursive)
 
 // TODO: this test could in principle also run without davix: need to figure out a way to detect if we have
 // remote access capabilities.
-#ifdef R__HAS_DAVIX
+#if defined(R__HAS_DAVIX) || defined(R__HAS_CURL)
 TEST(RFile, RemoteRead)
 {
    constexpr const char *kFileName = "https://root.cern/files/rootcode.root";

net/CMakeLists.txt

@@ -15,6 +15,10 @@ if(davix)
    add_subdirectory(davix)
 endif()
 
+if(curl)
+   add_subdirectory(curl)
+endif()
+
 if(netxng)
    add_subdirectory(netxng)
 endif()

net/curl/CMakeLists.txt

@@ -0,0 +1,28 @@
+# For the licensing terms see $ROOTSYS/LICENSE.
+# For the list of contributors see $ROOTSYS/README/CREDITS.
+
+############################################################################
+# CMakeLists.txt file for building ROOT net/curl package
+# @author Jakob Blomer, CERN
+############################################################################
+
+ROOT_STANDARD_LIBRARY_PACKAGE(RCurlHttp
+  HEADERS
+    ROOT/RCurlConnection.hxx
+    ROOT/RRawFileCurl.hxx
+    TCurlFile.h
+  SOURCES
+    src/RCurlConnection.cxx
+    src/RRawFileCurl.cxx
+    src/TCurlFile.cxx
+  LIBRARIES
+    CURL::libcurl
+  DEPENDENCIES
+    RIO
+)
+
+if(NOT MSVC)
+  target_compile_options(RCurlHttp PRIVATE -Wno-deprecated-declarations)
+endif()
+
+ROOT_ADD_TEST_SUBDIRECTORY(test)

net/curl/inc/LinkDef.h

@@ -0,0 +1,6 @@
+#ifdef __CLING__
+
+#pragma link C++ class ROOT::Internal::RRawFileCurl+;
+#pragma link C++ class TCurlFile+;
+
+#endif

net/curl/inc/ROOT/RCurlConnection.hxx

@@ -0,0 +1,102 @@
+// @(#)root/net:$Id$
+// Author: Jakob Blomer
+
+/*************************************************************************
+ * Copyright (C) 1995-2025, Rene Brun and Fons Rademakers.               *
+ * All rights reserved.                                                  *
+ *                                                                       *
+ * For the licensing terms see $ROOTSYS/LICENSE.                         *
+ * For the list of contributors see $ROOTSYS/README/CREDITS.             *
+ *************************************************************************/
+
+#ifndef ROOT_RCurlConnection
+#define ROOT_RCurlConnection
+
+#include <ROOT/RError.hxx>
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace ROOT {
+namespace Internal {
+
+/// Encapsulates a curl easy handle and provides an interface to send HTTP HEAD and (multi-)range queries.
+class RCurlConnection {
+public:
+   /// Return value for both HEAD and GET requests. In case of errors, provides the reason for the failure as code
+   /// and as message.
+   struct RStatus {
+      enum EStatusCode {
+         kSuccess = 0,
+         kTooManyRanges, ///< should not get to the user; number of request ranges is automatically reduced as needed
+         kNotFound,
+         kIOError,
+         kUnknown
+      };
+
+      EStatusCode fStatusCode = kUnknown;
+      std::string fStatusMsg;
+
+      RStatus() = default;
+      explicit RStatus(EStatusCode code) : fStatusCode(code) {}
+
+      explicit operator bool() const { return fStatusCode == kSuccess; }
+   };
+
+private:
+   void *fHandle = nullptr; ///< the CURL easy handle corresponding to this connection
+   /// If set to zero, automatically adjust: try with all given ranges and as long as the number of ranges is too large,
+   /// half it. If set to zero and automatic reduction of the number of requests is necessary, the number of requests
+   /// that works will be saved for further requests with this object.
+   std::size_t fMaxNRangesPerReqest = 0;
+   std::string fEscapedUrl;              ///< The URL provided in the constructor escaped according to standard rules
+   std::unique_ptr<char[]> fErrorBuffer; ///< For use by libcurl
+
+   void SetupErrorBuffer();
+   void SetOptions();
+   RResult<void> SetUrl(const std::string &url);
+   void Perform(RStatus &status);
+
+public:
+   /// Returned by SendHeadReq() if the HTTP response contains no content-length header
+   static constexpr std::uint64_t kUnknownSize = static_cast<std::uint64_t>(-1);
+
+   /// Caller-provided byte-range of the remote resource together with a pointer to a buffer.
+   struct RUserRange {
+      unsigned char *fDestination = nullptr;
+      std::uint64_t fOffset = 0;
+      std::size_t fLength = 0;
+      /// Usually equal to fLength for a successful call unless range goes out of the size of the remote resource
+      std::size_t fNBytesRecv = 0;
+
+      bool operator<(const RUserRange &other) const { return fOffset < other.fOffset; }
+   };
+
+   explicit RCurlConnection(const std::string &url);
+   ~RCurlConnection();
+   RCurlConnection(const RCurlConnection &other) = delete;
+   RCurlConnection &operator=(const RCurlConnection &other) = delete;
+   RCurlConnection(RCurlConnection &&other);
+   RCurlConnection &operator=(RCurlConnection &&other);
+
+   /// Checks if the resource exists and if it does, return the value of the content-length header as size
+   RStatus SendHeadReq(std::uint64_t &remoteSize);
+   /// Reads the given ranges from the remote resource. The ranges can be in any order and also overlapping. They
+   /// will be transformed in optimized HTTP ranges for a multi-range request. Ranges past the resource size are
+   /// valid (but won't receive any data). No limit on the number of ranges; if fMaxNRangesPerReqest is zero,
+   /// a valid batching of requests into multiple multi-range requests takes place automatically.
+   /// The fNBytesRecv member of the ranges is only well-defined on success.
+   RStatus SendRangesReq(std::size_t N, RUserRange *ranges);
+
+   const std::string &GetEscapedUrl() const { return fEscapedUrl; }
+
+   void SetMaxNRangesPerRequest(std::size_t val) { fMaxNRangesPerReqest = val; }
+   std::size_t GetMaxNRangesPerRequest() const { return fMaxNRangesPerReqest; }
+};
+
+} // namespace Internal
+} // namespace ROOT
+
+#endif

net/curl/inc/ROOT/RRawFileCurl.hxx

@@ -0,0 +1,52 @@
+// @(#)root/net:$Id$
+// Author: Jakob Blomer
+
+/*************************************************************************
+ * Copyright (C) 1995-2025, Rene Brun and Fons Rademakers.               *
+ * All rights reserved.                                                  *
+ *                                                                       *
+ * For the licensing terms see $ROOTSYS/LICENSE.                         *
+ * For the list of contributors see $ROOTSYS/README/CREDITS.             *
+ *************************************************************************/
+
+#ifndef ROOT_RRawFileCurl
+#define ROOT_RRawFileCurl
+
+#include <ROOT/RRawFile.hxx>
+
+#include <memory>
+
+namespace ROOT {
+namespace Internal {
+
+class RCurlConnection;
+
+/// \class RRawFileCurl
+/// \ingroup net
+///
+/// The RRawFileCurl class reads HTTP(S) resources using the curl library. The passed URL of the file
+/// needs to start with http:// or https://. The URL does not need to be escaped; URL encoding is handled internally.
+class RRawFileCurl : public RRawFile {
+private:
+   std::unique_ptr<RCurlConnection> fConnection;
+
+protected:
+   void OpenImpl() final;
+   size_t ReadAtImpl(void *buffer, size_t nbytes, std::uint64_t offset) final;
+   void ReadVImpl(RIOVec *ioVec, unsigned int nReq) final;
+   std::uint64_t GetSizeImpl() final;
+
+public:
+   static constexpr int kDefaultBlockSize = 128 * 1024; // relatively large 128k blocks for better network utilization
+
+   RRawFileCurl(std::string_view url, RRawFile::ROptions options);
+   ~RRawFileCurl();
+   std::unique_ptr<RRawFile> Clone() const final;
+
+   RCurlConnection &GetConnection();
+};
+
+} // namespace Internal
+} // namespace ROOT
+
+#endif