nvmeof: add DH-CHAP authentication support in NodeStageVolume

Implement DH-CHAP authentication for NVMe-oF initiator connections
with support for both unidirectional and bidirectional modes.

Changes:
- setupDHCHAPAuth(): Retrieve host/subsystem keys from KMS and configure
  ConnectRequest with authentication parameters
- Pass dhchapMode from volume context through connection flow
- Add --dhchap-secret and --dhchap-ctrl-secret to nvme connect command
- Support RBD metadata DEKStore (testing) and Vault KMS (production)

Authentication flow:
1. NodeStageVolume receives dhchapMode from volume context
2. Initialize SecurityKeyNVMEOFManager with same KMS as controller
3. Retrieve existing DH-CHAP keys (generated during ControllerPublishVolume)
4. For unidirectional: Add host key to nvme connect
5. For bidirectional: Add both host and subsystem keys

Signed-off-by: gadi-didi <gadi.didi@ibm.com>

Author: gadididi

internal/nvmeof/nodeserver/nodeserver.go

@@ -32,6 +32,7 @@ import (
 
 	csicommon "github.com/ceph/ceph-csi/internal/csi-common"
 	"github.com/ceph/ceph-csi/internal/nvmeof"
+	rbdutil "github.com/ceph/ceph-csi/internal/rbd"
 	"github.com/ceph/ceph-csi/internal/util"
 	"github.com/ceph/ceph-csi/internal/util/log"
 )
@@ -45,6 +46,12 @@ type NodeServer struct {
 	volumeLocks *util.IDLocker
 
 	initiator nvmeof.NVMeInitiator
+
+	// securityKeys manages DH-CHAP and PSK\TLS keys
+	securityKeys nvmeof.SecurityKeyManager
+
+	// node ID of this node server
+	nodeID string
 }
 
 // ConnectionInfo holds NVMe-oF connection details.
@@ -55,6 +62,7 @@ type NvmeConnectionInfo struct {
 	Listeners     []nvmeof.GatewayAddress `json:"listeners"`
 	HostNQN       string                  `json:"hostNQN,omitempty"`
 	Transport     string                  `json:"transport"`
+	DhchapMode    string                  `json:"dhchapMode,omitempty"`
 }
 
 // stageTransaction struct represents the state a transaction was when it either completed
@@ -76,6 +84,8 @@ const (
 	nvmeofListeners     = "listeners"
 	nvmeofHostNQN       = "HostNQN"
 	defaultTransport    = "tcp"
+	nvmeofdhchapMode    = "dhchapMode"
+	authenticationKMSID = "authenticationKMSID"
 )
 
 // NewNodeServer initialize a node server for ceph CSI driver.
@@ -90,6 +100,8 @@ func NewNodeServer(
 		DefaultNodeServer: *csicommon.NewDefaultNodeServer(d, t, "", map[string]string{}, map[string]string{}),
 		initiator:         nvmeInitiator,
 		volumeLocks:       util.NewIDLocker(),
+		securityKeys:      nil, // Initialize lazily when needed
+		nodeID:            nodeID,
 	}
 
 	// Load nvme kernel modules
@@ -141,13 +153,8 @@ func (ns *NodeServer) NodeStageVolume(
 	if err = util.ValidateNodeStageVolumeRequest(req); err != nil {
 		return nil, err
 	}
-
+	secrets := req.GetSecrets()
 	volumeID := req.GetVolumeId()
-	cr, err := util.NewUserCredentialsWithMigration(req.GetSecrets())
-	if err != nil {
-		return nil, status.Error(codes.InvalidArgument, err.Error())
-	}
-	defer cr.DeleteCredentials()
 	if acquired := ns.volumeLocks.TryAcquire(volumeID); !acquired {
 		log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID)
 
@@ -173,10 +180,13 @@ func (ns *NodeServer) NodeStageVolume(
 	if err != nil {
 		return nil, status.Errorf(codes.InvalidArgument, "invalid volume context: %v", err)
 	}
+	// Get authentication KMS ID from volume context.
+	// can be empty! it will take default KMS in that case
+	authKMSID := volumeContext[authenticationKMSID]
 
 	// perform the actual staging and if this fails, have undoStagingTransaction
 	// cleans up for us
-	txn, err := ns.stageTransaction(ctx, req, connectionInfo)
+	txn, err := ns.stageTransaction(ctx, req, connectionInfo, secrets, authKMSID)
 	defer func() {
 		if err != nil {
 			ns.undoStagingTransaction(ctx, req, txn)
@@ -426,13 +436,15 @@ func (ns *NodeServer) stageTransaction(
 	ctx context.Context,
 	req *csi.NodeStageVolumeRequest,
 	connectionInfo *NvmeConnectionInfo,
+	secrets map[string]string,
+	authKMSID string,
 ) (*stageTransaction, error) {
 	transaction := &stageTransaction{}
 
 	var err error
 
 	// perform the actual staging
-	devicePath, err := ns.connectToSubsystem(ctx, connectionInfo)
+	devicePath, err := ns.connectToSubsystem(ctx, req.GetVolumeId(), connectionInfo, secrets, authKMSID)
 	if err != nil {
 		return transaction, err
 	}
@@ -579,6 +591,10 @@ func (ns *NodeServer) getNvmeConnection(volumeContext, publishContext map[string
 	if !ok || hostNQN == "" {
 		return nil, errors.New("missing host NQN in publish context")
 	}
+	dhchapMode, ok := volumeContext[nvmeofdhchapMode]
+	if !ok || dhchapMode == "" || dhchapMode == "none" {
+		dhchapMode = ""
+	}
 
 	return &NvmeConnectionInfo{
 		SubsystemNQN:  subsystemNQN,
@@ -587,11 +603,18 @@ func (ns *NodeServer) getNvmeConnection(volumeContext, publishContext map[string
 		Listeners:     listeners,
 		HostNQN:       hostNQN,
 		Transport:     defaultTransport,
+		DhchapMode:    dhchapMode,
 	}, nil
 }
 
 // connectToSubsystem connects to the NVMe-oF subsystem and returns device path.
-func (ns *NodeServer) connectToSubsystem(ctx context.Context, info *NvmeConnectionInfo) (string, error) {
+func (ns *NodeServer) connectToSubsystem(
+	ctx context.Context,
+	volumeID string,
+	info *NvmeConnectionInfo,
+	secrets map[string]string,
+	authKMSID string,
+) (string, error) {
 	// Create connect request
 	connectReq := &nvmeof.ConnectRequest{
 		SubsystemNQN: info.SubsystemNQN,
@@ -600,6 +623,13 @@ func (ns *NodeServer) connectToSubsystem(ctx context.Context, info *NvmeConnecti
 		HostNQN:      info.HostNQN,
 	}
 
+	// Setup DH-CHAP authentication if required
+	if info.DhchapMode != nvmeof.DHCHAPEmpty && info.DhchapMode != nvmeof.DHCHAPModeNone {
+		if err := ns.setupDHCHAPAuth(ctx, volumeID, info, secrets, authKMSID, connectReq); err != nil {
+			return "", err
+		}
+	}
+
 	// Connect to subsystem
 	_, err := ns.initiator.ConnectSubsystem(ctx, connectReq)
 	if err != nil {
@@ -732,3 +762,75 @@ func (ns *NodeServer) getDeviceFromMount(ctx context.Context, mountPath string)
 
 	return "", fmt.Errorf("no mount found for path %s", mountPath)
 }
+
+func (cs *NodeServer) getOrInitSecurityKeys(
+	ctx context.Context,
+	kmsID string,
+	credentials map[string]string,
+) (nvmeof.SecurityKeyManager, error) {
+	if cs.securityKeys != nil {
+		return cs.securityKeys, nil
+	}
+
+	var err error
+	securityKeys, err := nvmeof.InitSecurityKeyManager(ctx, kmsID, credentials)
+
+	// Only cache if it doesn't need external DEKStore (like Vault)
+	// (RBD Metadata KMS needs fresh RBD volume each call)
+	if !errors.Is(err, nvmeof.ErrDEKStoreNeeded) {
+		cs.securityKeys = securityKeys
+	}
+
+	return securityKeys, err
+}
+
+// setupDHCHAPAuth configures DH-CHAP authentication for the connection.
+func (ns *NodeServer) setupDHCHAPAuth(
+	ctx context.Context,
+	volumeID string,
+	info *NvmeConnectionInfo,
+	secrets map[string]string,
+	authKMSID string,
+	connectReq *nvmeof.ConnectRequest,
+) error {
+	// Initialize security key manager
+	securityKeys, err := ns.getOrInitSecurityKeys(ctx, authKMSID, secrets)
+
+	// Setup DEK store if needed (for metadata KMS)
+	if errors.Is(err, nvmeof.ErrDEKStoreNeeded) {
+		cr, err := util.NewUserCredentialsWithMigration(secrets)
+		if err != nil {
+			return fmt.Errorf("failed to get user credentials: %w", err)
+		}
+		defer cr.DeleteCredentials()
+
+		rbdVol, err := rbdutil.GenVolFromVolID(ctx, volumeID, cr, secrets)
+		if err != nil {
+			return fmt.Errorf("failed to get volume: %w", err)
+		}
+		defer rbdVol.Destroy(ctx)
+
+		dekStore := nvmeof.NewRBDVolumeDEKStore(rbdVol)
+		securityKeys.SetDEKStore(dekStore)
+	} else if err != nil {
+		return fmt.Errorf("failed to initialize security key manager: %w", err)
+	}
+
+	// Get host key
+	hostKey, err := nvmeof.GetOrCreateDHCHAPHostKey(ctx, securityKeys, ns.nodeID, info.SubsystemNQN)
+	if err != nil {
+		return fmt.Errorf("failed to get DHCHAP host key: %w", err)
+	}
+	connectReq.HostDhchapKey = hostKey
+
+	// Get subsystem key for bidirectional mode
+	if info.DhchapMode == nvmeof.DHCHAPModeBiDirectional {
+		subsystemKey, err := nvmeof.GetOrCreateDHCHAPSubsystemKey(ctx, securityKeys, ns.nodeID, info.SubsystemNQN)
+		if err != nil {
+			return fmt.Errorf("failed to get DH-CHAP subsystem key: %w", err)
+		}
+		connectReq.SubsystemDhchapKey = subsystemKey
+	}
+
+	return nil
+}

internal/nvmeof/nvmeof_initiator.go

@@ -61,6 +61,10 @@ type ConnectRequest struct {
 	Listeners    []GatewayAddress
 	Transport    string // "tcp"
 	HostNQN      string // Optional - empty means use system default
+	// Optional - In-band authentication controller secret for bi-directional authentication.
+	SubsystemDhchapKey string
+	// Optional - In-band authentication secret for uni-directional authentication
+	HostDhchapKey string
 }
 
 // nvmeInitiator implements NVMeInitiator interface.
@@ -185,7 +189,14 @@ func (ni *nvmeInitiator) ConnectSubsystem(ctx context.Context, req *ConnectReque
 		if req.HostNQN != "" {
 			args = append(args, "--hostnqn", req.HostNQN)
 		}
-
+		// if Host DH-CHAP key is provided, add it to the command
+		if req.HostDhchapKey != "" {
+			args = append(args, "--dhchap-secret", req.HostDhchapKey)
+		}
+		// if Subsystem DH-CHAP key is provided, add it to the command (for bi-directional auth)
+		if req.SubsystemDhchapKey != "" {
+			args = append(args, "--dhchap-ctrl-secret", req.SubsystemDhchapKey)
+		}
 		stdout, stderr, err := util.ExecCommandWithTimeout(ctx, connectTimeout, "nvme", args...)
 		// Execute connection
 		if err != nil {