Optimize cache miss path with deferred cache write (configurable)

Add configuration option proxy.config.http.cache.defer_write_on_miss
(default: 0/disabled) to defer opening the cache for write until after
response headers are received and evaluated for cacheability.

When enabled, this optimization:
- Avoids unnecessary cache overhead for non-cacheable responses
- Can improve throughput ~21x for non-cacheable content

Trade-offs when enabled:
- May affect read-while-write functionality
- May reduce request coalescing for popular uncached URLs
- Recommended only for traffic patterns with mostly unique URLs
  or predominantly non-cacheable content

The optimization works by:
1. On cache miss, set cache_open_write_deferred=true instead of
   immediately opening cache for write
2. Go to origin and fetch response headers
3. Check if response is cacheable:
   - If not cacheable (no-store, etc.): skip cache write entirely
   - If cacheable: open cache for write and proceed normally

Author: bryancall

configs/records.yaml.default.in

@@ -86,6 +86,12 @@ records:
 # https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#proxy-config-http-cache-when-to-revalidate
       when_to_revalidate: 0
 
+# Defer cache open write until response headers are received.
+# Improves performance for non-cacheable responses but may affect
+# read-while-write and request coalescing for popular uncached URLs.
+# 0 - disabled (default, safe), 1 - enabled
+      defer_write_on_miss: 0
+
 ##############################################################################
 # Origin server connect attempts. Docs:
 #    https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#origin-server-connect-attempts

include/proxy/http/HttpConfig.h

@@ -578,6 +578,11 @@ struct OverridableHttpConfigParams {
 
   MgmtByte cache_open_write_fail_action = 0;
 
+  ////////////////////////////////////////////////
+  // Defer cache open write until response hdrs //
+  ////////////////////////////////////////////////
+  MgmtByte cache_defer_write_on_miss = 0;
+
   ////////////////////////
   // Check Post request //
   ////////////////////////

include/proxy/http/HttpTransact.h

@@ -693,6 +693,10 @@ class HttpTransact
     bool is_method_stats_incremented  = false;
     bool skip_ip_allow_yaml           = false;
 
+    /// True if we deferred opening the cache for write until after response headers.
+    /// This is an optimization to avoid cache overhead for non-cacheable responses.
+    bool cache_open_write_deferred = false;
+
     /// True if the response is cacheable because of negative caching configuration.
     ///
     /// This being true implies the following:
@@ -997,6 +1001,7 @@ class HttpTransact
   static void set_cache_prepare_write_action_for_new_request(State *s);
   static void build_response_from_cache(State *s, HTTPWarningCode warning_code);
   static void handle_cache_write_lock(State *s);
+  static void handle_deferred_cache_write_complete(State *s);
   static void HandleResponse(State *s);
   static void HandleUpdateCachedObject(State *s);
   static void HandleUpdateCachedObjectContinue(State *s);

src/proxy/http/HttpConfig.cc

@@ -1087,6 +1087,7 @@ HttpConfig::startup()
   HttpEstablishStaticConfigByte(c.disallow_post_100_continue, "proxy.config.http.disallow_post_100_continue");
 
   HttpEstablishStaticConfigByte(c.oride.cache_open_write_fail_action, "proxy.config.http.cache.open_write_fail_action");
+  HttpEstablishStaticConfigByte(c.oride.cache_defer_write_on_miss, "proxy.config.http.cache.defer_write_on_miss");
 
   HttpEstablishStaticConfigByte(c.oride.cache_when_to_revalidate, "proxy.config.http.cache.when_to_revalidate");
   HttpEstablishStaticConfigByte(c.oride.cache_required_headers, "proxy.config.http.cache.required_headers");
@@ -1391,6 +1392,7 @@ HttpConfig::reconfigure()
               params->oride.max_cache_open_write_retries);
     }
   }
+  params->oride.cache_defer_write_on_miss = m_master.oride.cache_defer_write_on_miss;
 
   params->oride.cache_when_to_revalidate = m_master.oride.cache_when_to_revalidate;
   params->max_post_size                  = m_master.max_post_size;

src/proxy/http/HttpTransact.cc

@@ -3231,6 +3231,62 @@ HttpTransact::handle_cache_write_lock(State *s)
   }
 }
 
+///////////////////////////////////////////////////////////////////////////////
+// Name       : handle_deferred_cache_write_complete
+// Description: Called after cache write opens for deferred cache write.
+//              We already have the server response headers, so we continue
+//              with cache write handling.
+//
+// Possible Next States From Here:
+// - handle_cache_operation_on_forward_server_response (if cache write succeeded)
+// - handle_no_cache_operation_on_forward_server_response (if cache write failed)
+//
+///////////////////////////////////////////////////////////////////////////////
+void
+HttpTransact::handle_deferred_cache_write_complete(State *s)
+{
+  TxnDbg(dbg_ctl_http_trans, "handle_deferred_cache_write_complete");
+  TxnDbg(dbg_ctl_http_seq, "Entering handle_deferred_cache_write_complete");
+
+  ink_assert(s->cache_info.action == CacheAction_t::PREPARE_TO_WRITE);
+
+  switch (s->cache_info.write_lock_state) {
+  case CacheWriteLock_t::SUCCESS:
+    // Cache write lock acquired successfully
+    TxnDbg(dbg_ctl_http_trans, "[hdcwc] cache write lock acquired");
+    SET_UNPREPARE_CACHE_ACTION(s->cache_info);
+    // Now we have WRITE action, handle the response with caching
+    handle_cache_operation_on_forward_server_response(s);
+    return;
+
+  case CacheWriteLock_t::FAIL:
+    // Could not get cache write lock, continue without caching
+    TxnDbg(dbg_ctl_http_trans, "[hdcwc] cache write lock failed, continuing without cache");
+    Metrics::Counter::increment(http_rsb.cache_open_write_fail_count);
+    s->cache_info.action       = CacheAction_t::NO_ACTION;
+    s->cache_info.write_status = CacheWriteStatus_t::LOCK_MISS;
+    break;
+
+  case CacheWriteLock_t::READ_RETRY:
+    // This shouldn't happen for deferred write since we don't have an object to read
+    TxnDbg(dbg_ctl_http_trans, "[hdcwc] unexpected READ_RETRY for deferred write");
+    s->cache_info.action       = CacheAction_t::NO_ACTION;
+    s->cache_info.write_status = CacheWriteStatus_t::LOCK_MISS;
+    break;
+
+  case CacheWriteLock_t::INIT:
+  default:
+    ink_release_assert(0);
+    break;
+  }
+
+  // If we get here, cache write failed - continue without caching
+  // The original handle_no_cache_operation_on_forward_server_response will be called
+  // but we need to skip the deferred cache write check since we just tried it
+  // s->cache_open_write_deferred is already false from the first call
+  handle_no_cache_operation_on_forward_server_response(s);
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 // Name       : HandleCacheOpenReadMiss
 // Description: cache looked up, miss or hit, but needs authorization
@@ -3276,6 +3332,13 @@ HttpTransact::HandleCacheOpenReadMiss(State *s)
     s->cache_info.action = CacheAction_t::NO_ACTION;
   } else if (s->api_server_response_no_store) { // plugin may have decided not to cache the response
     s->cache_info.action = CacheAction_t::NO_ACTION;
+  } else if (s->txn_conf->cache_defer_write_on_miss) {
+    // Defer cache open write until after response headers are received.
+    // This avoids cache overhead for non-cacheable responses but may
+    // affect read-while-write and request coalescing for popular URLs.
+    s->cache_open_write_deferred = true;
+    s->cache_info.action         = CacheAction_t::NO_ACTION;
+    TxnDbg(dbg_ctl_http_trans, "[HandleCacheOpenReadMiss] deferring cache open write until response");
   } else {
     HttpTransact::set_cache_prepare_write_action_for_new_request(s);
   }
@@ -4677,6 +4740,24 @@ HttpTransact::handle_no_cache_operation_on_forward_server_response(State *s)
   TxnDbg(dbg_ctl_http_trans, "(hncoofsr)");
   TxnDbg(dbg_ctl_http_seq, "Entering handle_no_cache_operation_on_forward_server_response");
 
+  // Check if we deferred the cache open write and the response is cacheable.
+  // If so, we need to open the cache for write now.
+  if (s->cache_open_write_deferred && s->txn_conf->cache_http) {
+    bool cacheable = is_response_cacheable(s, &s->hdr_info.client_request, &s->hdr_info.server_response);
+    TxnDbg(dbg_ctl_http_trans, "[hncoofsr] deferred cache write, response %s cacheable", cacheable ? "is" : "is not");
+
+    if (cacheable) {
+      // Response is cacheable - open the cache for write now
+      s->cache_open_write_deferred   = false;
+      s->cache_info.action           = CacheAction_t::PREPARE_TO_WRITE;
+      s->cache_info.write_lock_state = CacheWriteLock_t::INIT;
+      // After cache write opens, continue with handle_cache_operation_on_forward_server_response
+      TRANSACT_RETURN(StateMachineAction_t::CACHE_ISSUE_WRITE, handle_deferred_cache_write_complete);
+    }
+    // Not cacheable - continue with no-cache operation
+    s->cache_open_write_deferred = false;
+  }
+
   bool        keep_alive = s->current.server->keep_alive == HTTPKeepAlive::KEEPALIVE;
   const char *warn_text  = nullptr;