From ee6819254d66ef2f52b4fb3f9fa0528c2c3e37a4 Mon Sep 17 00:00:00 2001 From: Roxy Light Date: Fri, 24 Oct 2025 19:33:38 -0700 Subject: [PATCH] Store signatures in backend database and record them during realization Add command-line flag to configure keyring. Fixes #159 --- cmd/zb/keys.go | 176 +++++++++++++++ cmd/zb/main.go | 1 + cmd/zb/serve.go | 7 + internal/backend/backend.go | 52 +++++ internal/backend/backend_store.go | 155 ++++++++++++- internal/backend/realize.go | 20 +- internal/backend/realize_test.go | 203 ++++++++++++++++-- internal/backend/sql/build/insert_result.sql | 4 + .../backend/sql/build/result_signatures.sql | 20 ++ internal/backend/sql/build/results.sql | 3 + internal/backend/sql/insert_public_key.sql | 3 + internal/backend/sql/insert_signature.sql | 13 ++ internal/backend/sql/schema/04.sql | 24 +++ internal/zbstorerpc/zbstorerpc.go | 11 +- 14 files changed, 643 insertions(+), 49 deletions(-) create mode 100644 cmd/zb/keys.go create mode 100644 internal/backend/sql/build/result_signatures.sql create mode 100644 internal/backend/sql/insert_public_key.sql create mode 100644 internal/backend/sql/insert_signature.sql create mode 100644 internal/backend/sql/schema/04.sql diff --git a/cmd/zb/keys.go b/cmd/zb/keys.go new file mode 100644 index 0000000..dc5867c --- /dev/null +++ b/cmd/zb/keys.go @@ -0,0 +1,176 @@ +// Copyright 2025 The zb Authors +// SPDX-License-Identifier: MIT + +package main + +import ( + "cmp" + "context" + "crypto/ed25519" + "fmt" + "io" + "os" + + jsonv2 "github.com/go-json-experiment/json" + "github.com/go-json-experiment/json/jsontext" + "github.com/spf13/cobra" + "zb.256lights.llc/pkg/internal/backend" + "zb.256lights.llc/pkg/zbstore" +) + +type privateKeyFile struct { + Format zbstore.RealizationSignatureFormat `json:"format"` + Key []byte `json:"key,format:base64"` +} + +func (f *privateKeyFile) appendToKeyring(dst *backend.Keyring) error { + switch f.Format { + case zbstore.Ed25519SignatureFormat: + if got, want := len(f.Key), ed25519.SeedSize; got != want { + return fmt.Errorf("key is wrong size (decoded is %d instead of %d bytes)", got, want) + } + dst.Ed25519 = append(dst.Ed25519, ed25519.NewKeyFromSeed(f.Key)) + default: + return fmt.Errorf("unknown format %q", f.Format) + } + return nil +} + +func readKeyringFromFiles(files []string) (*backend.Keyring, error) { + result := new(backend.Keyring) + for _, path := range files { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var parsed privateKeyFile + if err := jsonv2.Unmarshal(data, &parsed); err != nil { + return nil, fmt.Errorf("read %s: %v", path, err) + } + if err := parsed.appendToKeyring(result); err != nil { + return nil, fmt.Errorf("read %s: %v", path, err) + } + } + return result, nil +} + +func newKeyCommand() *cobra.Command { + c := &cobra.Command{ + Use: "key COMMAND", + Short: "operate on signing key files", + DisableFlagsInUseLine: true, + SilenceErrors: true, + SilenceUsage: true, + } + c.AddCommand( + newGenerateKeyCommand(), + newShowPublicKeyCommand(), + ) + return c +} + +func newGenerateKeyCommand() *cobra.Command { + c := &cobra.Command{ + Use: "generate [-o PATH]", + Short: "generate a new signing key", + DisableFlagsInUseLine: true, + Args: cobra.NoArgs, + SilenceErrors: true, + SilenceUsage: true, + } + outputPath := c.Flags().StringP("output", "o", "", "`file` to write to (default is stdout)") + c.RunE = func(cmd *cobra.Command, args []string) error { + outputFile := os.Stdout + if *outputPath != "" { + var err error + outputFile, err = os.Create(*outputPath) + if err != nil { + return err + } + } + err1 := runGenerateKey(cmd.Context(), outputFile) + var err2 error + if *outputPath != "" { + err2 = outputFile.Close() + } + return cmp.Or(err1, err2) + } + return c +} + +func runGenerateKey(ctx context.Context, dst io.Writer) error { + _, newKey, err := ed25519.GenerateKey(nil) + if err != nil { + return err + } + keyFile := &privateKeyFile{ + Format: zbstore.Ed25519SignatureFormat, + Key: newKey.Seed(), + } + keyFileData, err := jsonv2.Marshal(keyFile, jsontext.Multiline(true)) + if err != nil { + return err + } + keyFileData = append(keyFileData, '\n') + _, err = dst.Write(keyFileData) + return err +} + +func newShowPublicKeyCommand() *cobra.Command { + c := &cobra.Command{ + Use: "show-public [PATH [...]]", + Short: "print public key of signing keys", + DisableFlagsInUseLine: true, + Args: cobra.ArbitraryArgs, + SilenceErrors: true, + SilenceUsage: true, + } + c.RunE = func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + if len(args) == 0 { + return runShowPublicKey(ctx, os.Stdout, os.Stdin) + } + for _, path := range args { + f, err := os.Open(path) + if err != nil { + return err + } + err = runShowPublicKey(ctx, os.Stdout, f) + f.Close() + if err != nil { + return err + } + } + return nil + } + return c +} + +func runShowPublicKey(ctx context.Context, dst io.Writer, src io.Reader) error { + keyFile := new(privateKeyFile) + if err := jsonv2.UnmarshalRead(src, keyFile, jsonv2.RejectUnknownMembers(false)); err != nil { + return err + } + k := new(backend.Keyring) + if err := keyFile.appendToKeyring(k); err != nil { + return err + } + var result struct { + Format zbstore.RealizationSignatureFormat `json:"format"` + PublicKey []byte `json:"publicKey,format:base64"` + } + switch { + case len(k.Ed25519) > 0: + result.Format = zbstore.Ed25519SignatureFormat + result.PublicKey = k.Ed25519[0].Public().(ed25519.PublicKey) + default: + return nil + } + data, err := jsonv2.Marshal(result, jsontext.Multiline(true)) + if err != nil { + return err + } + data = append(data, '\n') + _, err = dst.Write(data) + return err +} diff --git a/cmd/zb/main.go b/cmd/zb/main.go index 7a5fbb7..28eba8c 100644 --- a/cmd/zb/main.go +++ b/cmd/zb/main.go @@ -95,6 +95,7 @@ func main() { newBuildCommand(g), newDerivationCommand(g), newEvalCommand(g), + newKeyCommand(), newNARCommand(), newServeCommand(g), newStoreCommand(g), diff --git a/cmd/zb/serve.go b/cmd/zb/serve.go index bf99235..6970a60 100644 --- a/cmd/zb/serve.go +++ b/cmd/zb/serve.go @@ -46,6 +46,7 @@ type serveOptions struct { buildDir string buildUsersGroup string logDir string + keyFiles []string sandbox bool sandboxPaths map[string]backend.SandboxPath allowKeepFailed bool @@ -82,6 +83,7 @@ func newServeCommand(g *globalConfig) *cobra.Command { c.Flags().StringVar(&opts.buildDir, "build-root", os.TempDir(), "`dir`ectory to store temporary build artifacts") c.Flags().StringVar(&opts.logDir, "log-directory", filepath.Join(filepath.Dir(string(zbstore.DefaultDirectory())), "var", "log", "zb"), "`dir`ectory to store builder logs in") c.Flags().StringVar(&opts.buildUsersGroup, "build-users-group", opts.buildUsersGroup, "name of Unix `group` of users to run builds as") + c.Flags().StringArrayVar(&opts.keyFiles, "signing-key", nil, "key `file` for signing realizations (can be passed multiple times)") c.Flags().BoolVar(&opts.sandbox, "sandbox", opts.sandbox, "run builders in a restricted environment") sandboxPaths := make(map[string]string) c.Flags().Var(pathMapFlag(sandboxPaths), "sandbox-path", "`path` to allow in sandbox (can be passed multiple times)") @@ -113,6 +115,10 @@ func runServe(ctx context.Context, g *globalConfig, opts *serveOptions) error { } return fmt.Errorf("sandboxing requested but unable to use (are you running with admin privileges?)") } + keyring, err := readKeyringFromFiles(opts.keyFiles) + if err != nil { + return err + } storeDirGroupID, buildUsers, err := buildUsersForGroup(ctx, opts.buildUsersGroup) if err != nil { return err @@ -203,6 +209,7 @@ func runServe(ctx context.Context, g *globalConfig, opts *serveOptions) error { AllowKeepFailed: opts.allowKeepFailed, CoresPerBuild: opts.coresPerBuild, BuildLogRetention: opts.buildLogRetention, + Keyring: keyring, }) defer func() { if err := backendServer.Close(); err != nil { diff --git a/internal/backend/backend.go b/internal/backend/backend.go index e0fded3..9b4bc22 100644 --- a/internal/backend/backend.go +++ b/internal/backend/backend.go @@ -6,6 +6,7 @@ package backend import ( "context" + "crypto/ed25519" "errors" "fmt" "io" @@ -83,6 +84,10 @@ type Options struct { // BuildLogRetention is the length of time to retain build logs. // If non-positive, then build logs will be not be automatically deleted. BuildLogRetention time.Duration + + // Keyring is a set of keys that will be used to sign realizations + // that this server realizes. + Keyring *Keyring } // A SandboxPath is the set of options for SandboxPaths in [Options]. @@ -128,6 +133,7 @@ type Server struct { db *sqlitemigration.Pool allowKeepFailed bool buildContext func(context.Context, string) context.Context + keyring *Keyring sandbox bool sandboxPaths map[string]SandboxPath @@ -169,6 +175,7 @@ func NewServer(dir zbstore.Directory, dbPath string, opts *Options) *Server { users: users, activeBuilds: make(map[uuid.UUID]context.CancelFunc), buildContext: opts.BuildContext, + keyring: opts.Keyring.Clone(), db: sqlitemigration.NewPool(dbPath, loadSchema(), sqlitemigration.Options{ Flags: sqlite.OpenCreate | sqlite.OpenReadWrite, @@ -894,6 +901,51 @@ func (s *Server) optimizeDatabase(ctx context.Context) { } } +// A Keyring is a set of private keys to use for signing. +// Nil or the zero value is an empty set of keys. +type Keyring struct { + Ed25519 []ed25519.PrivateKey +} + +// Clone returns a new keyring with contents identical to k. +// If k is nil, then Clone returns nil. +func (k *Keyring) Clone() *Keyring { + if k == nil { + return nil + } + k2 := new(Keyring) + if len(k.Ed25519) > 0 { + k2.Ed25519 = make([]ed25519.PrivateKey, len(k.Ed25519)) + for i, key := range k.Ed25519 { + k2.Ed25519[i] = slices.Clone(key) + } + } + return k2 +} + +// Sign creates signatures for the realization using all the private keys in the keyring. +func (k *Keyring) Sign(ref zbstore.RealizationOutputReference, r *zbstore.Realization) ([]*zbstore.RealizationSignature, error) { + n := 0 + if k != nil { + n = len(k.Ed25519) + } + if n == 0 { + return nil, nil + } + + result := make([]*zbstore.RealizationSignature, 0, n) + var returnedError error + for _, key := range k.Ed25519 { + sig, err := zbstore.SignRealizationWithEd25519(ref, r, key) + if err != nil { + returnedError = errors.Join(returnedError, err) + continue + } + result = append(result, sig) + } + return result, returnedError +} + func parseBuildID(id string) (_ uuid.UUID, ok bool) { u, err := uuid.Parse(id) if err != nil || id != u.String() { diff --git a/internal/backend/backend_store.go b/internal/backend/backend_store.go index e82cc13..dee4073 100644 --- a/internal/backend/backend_store.go +++ b/internal/backend/backend_store.go @@ -167,7 +167,7 @@ type realizationOutput struct { references map[zbstore.Path]sets.Set[equivalenceClass] } -func recordRealizations(ctx context.Context, conn *sqlite.Conn, drvHash nix.Hash, outputs map[string]realizationOutput) (err error) { +func recordRealizations(ctx context.Context, conn *sqlite.Conn, keyring *Keyring, drvHash nix.Hash, outputs map[string]realizationOutput) (err error) { if log.IsEnabled(log.Debug) { outputPaths := make(map[string]zbstore.Path) for outputName, out := range outputs { @@ -208,6 +208,16 @@ func recordRealizations(ctx context.Context, conn *sqlite.Conn, drvHash nix.Hash return fmt.Errorf("record realizations for %s: %v", drvHash, err) } defer refClassStmt.Finalize() + publicKeyStmt, err := sqlitex.PrepareTransientFS(conn, sqlFiles(), "insert_public_key.sql") + if err != nil { + return fmt.Errorf("record realizations for %s: %v", drvHash, err) + } + defer publicKeyStmt.Finalize() + signatureStmt, err := sqlitex.PrepareTransientFS(conn, sqlFiles(), "insert_signature.sql") + if err != nil { + return fmt.Errorf("record realizations for %s: %v", drvHash, err) + } + defer signatureStmt.Finalize() realizationStmt.SetText(":drv_hash_algorithm", drvHash.Type().String()) realizationStmt.SetBytes(":drv_hash_bits", drvHash.Bytes(nil)) @@ -247,11 +257,63 @@ func recordRealizations(ctx context.Context, conn *sqlite.Conn, drvHash nix.Hash } } } + + ref := zbstore.RealizationOutputReference{ + DerivationHash: drvHash, + OutputName: outputName, + } + signatures, err := keyring.Sign(ref, makeRealization(output)) + if err != nil { + log.Warnf(ctx, "%v", err) + } + for _, sig := range signatures { + publicKeyStmt.SetText(":format", string(sig.Format)) + publicKeyStmt.SetBytes(":public_key", sig.PublicKey) + if _, err := publicKeyStmt.Step(); err != nil { + return fmt.Errorf("record realizations for %v: %v", ref, err) + } + if err := publicKeyStmt.Reset(); err != nil { + return fmt.Errorf("record realizations for %v: %v", ref, err) + } + + signatureStmt.SetText(":drv_hash_algorithm", drvHash.Type().String()) + signatureStmt.SetBytes(":drv_hash_bits", drvHash.Bytes(nil)) + signatureStmt.SetText(":output_name", outputName) + signatureStmt.SetText(":output_path", string(output.path)) + signatureStmt.SetText(":format", string(sig.Format)) + signatureStmt.SetBytes(":public_key", sig.PublicKey) + signatureStmt.SetBytes(":signature", sig.Signature) + if _, err := signatureStmt.Step(); err != nil { + return fmt.Errorf("record realizations for %v: %v", ref, err) + } + if err := signatureStmt.Reset(); err != nil { + return fmt.Errorf("record realizations for %v: %v", ref, err) + } + } } return nil } +func makeRealization(output realizationOutput) *zbstore.Realization { + r := &zbstore.Realization{ + OutputPath: output.path, + } + for path, eqClasses := range output.references { + for eqClass := range eqClasses.All() { + rc := &zbstore.ReferenceClass{Path: path} + if !eqClass.isZero() { + rc.Realization = zbstore.NonNull(zbstore.RealizationOutputReference{ + DerivationHash: eqClass.drvHashKey.toHash(), + OutputName: eqClass.outputName.Value(), + }) + } + r.ReferenceClasses = append(r.ReferenceClasses, rc) + } + } + return r +} + // pathInfo returns basic information about an object in the store. func pathInfo(conn *sqlite.Conn, path zbstore.Path) (_ *ObjectInfo, err error) { defer sqlitex.Save(conn)(&err) @@ -647,16 +709,21 @@ func recordExpandResult(conn *sqlite.Conn, buildID uuid.UUID, result *zbstorerpc return nil } -func insertBuildResult(conn *sqlite.Conn, buildID uuid.UUID, drvPath zbstore.Path, t time.Time) (buildResultID int64, err error) { +func insertBuildResult(conn *sqlite.Conn, buildID uuid.UUID, drvPath zbstore.Path, drvHash nix.Hash, t time.Time) (buildResultID int64, err error) { defer sqlitex.Save(conn)(&err) if err := upsertPath(conn, drvPath); err != nil { return -1, fmt.Errorf("record build result for %s in %v: %v", drvPath, buildID, err) } + if err := upsertDrvHash(conn, drvHash); err != nil { + return -1, fmt.Errorf("record build result for %s in %v: %v", drvPath, buildID, err) + } err = sqlitex.ExecuteTransientFS(conn, sqlFiles(), "build/insert_result.sql", &sqlitex.ExecOptions{ Named: map[string]any{ - ":build_id": buildID.String(), - ":drv_path": string(drvPath), - ":timestamp_millis": t.UnixMilli(), + ":build_id": buildID.String(), + ":drv_path": string(drvPath), + ":drv_hash_algorithm": drvHash.Type().String(), + ":drv_hash_bits": drvHash.Bytes(nil), + ":timestamp_millis": t.UnixMilli(), }, ResultFunc: func(stmt *sqlite.Stmt) error { buildResultID = stmt.ColumnInt64(0) @@ -672,9 +739,16 @@ func insertBuildResult(conn *sqlite.Conn, buildID uuid.UUID, drvPath zbstore.Pat // findBuildResults appends the build results in the build with the given ID to dst. // If drvPath is not empty, then only the result for drvPath is appended (if it exists). // If logDir is not empty, then the LogSize field in [*zbstorerpc.BuildResult] will be populated. -func findBuildResults(dst []*zbstorerpc.BuildResult, conn *sqlite.Conn, logDir string, buildID uuid.UUID, drvPath zbstore.Path) ([]*zbstorerpc.BuildResult, error) { +func findBuildResults(dst []*zbstorerpc.BuildResult, conn *sqlite.Conn, logDir string, buildID uuid.UUID, drvPath zbstore.Path) (_ []*zbstorerpc.BuildResult, err error) { + defer sqlitex.Save(conn)(&err) + + signatureStmt, err := sqlitex.PrepareTransientFS(conn, sqlFiles(), "build/result_signatures.sql") + if err != nil { + return dst, fmt.Errorf("list build results for %v: %v", buildID, err) + } + defer signatureStmt.Finalize() initDstLen := len(dst) - err := sqlitex.ExecuteTransientFS(conn, sqlFiles(), "build/results.sql", &sqlitex.ExecOptions{ + err = sqlitex.ExecuteTransientFS(conn, sqlFiles(), "build/results.sql", &sqlitex.ExecOptions{ Named: map[string]any{ ":build_id": buildID.String(), ":drv_path": string(drvPath), @@ -688,8 +762,20 @@ func findBuildResults(dst []*zbstorerpc.BuildResult, conn *sqlite.Conn, logDir s if len(dst) > initDstLen && dst[len(dst)-1].DrvPath == drvPath { curr = dst[len(dst)-1] } else { + var drvHash nix.Hash + if algo := stmt.GetText("drv_hash_algorithm"); algo != "" { + buf := make([]byte, stmt.GetLen("drv_hash_bits")) + stmt.GetBytes("drv_hash_bits", buf) + var err error + drvHash, err = unmarshalHash(algo, buf) + if err != nil { + return err + } + } + curr = &zbstorerpc.BuildResult{ DrvPath: drvPath, + DrvHash: zbstore.NonNull(drvHash), Status: zbstorerpc.BuildStatus(stmt.GetText("status")), Outputs: []*zbstorerpc.RealizeOutput{}, } @@ -715,6 +801,13 @@ func findBuildResults(dst []*zbstorerpc.BuildResult, conn *sqlite.Conn, logDir s return fmt.Errorf("output %s: %v", outputName, err) } newOutput.Path = zbstorerpc.NonNull(p) + + newOutput.Signatures, err = signaturesForRealization(signatureStmt, buildID, drvPath, outputName, p) + if err != nil { + // signaturesForRealization includes the outputName in the error message, + // so no need to additionally wrap. + return err + } } curr.Outputs = append(curr.Outputs, newOutput) } @@ -731,6 +824,42 @@ func findBuildResults(dst []*zbstorerpc.BuildResult, conn *sqlite.Conn, logDir s return dst, nil } +// signaturesForRealization fetches the list of signatures stored for the given realization. +func signaturesForRealization(stmt *sqlite.Stmt, buildID uuid.UUID, drvPath zbstore.Path, outputName string, outputPath zbstore.Path) ([]*zbstore.RealizationSignature, error) { + var result []*zbstore.RealizationSignature + stmt.SetText(":build_id", buildID.String()) + stmt.SetText(":drv_path", string(drvPath)) + stmt.SetText(":output_name", outputName) + stmt.SetText(":output_path", string(outputPath)) + + for { + hasRow, err := stmt.Step() + if err != nil { + _ = stmt.Reset() + return nil, fmt.Errorf("output %s: signatures: %v", outputName, err) + } + if !hasRow { + break + } + + buf := make([]byte, stmt.GetLen("public_key")+stmt.GetLen("signature")) + newSignature := &zbstore.RealizationSignature{ + Format: zbstore.RealizationSignatureFormat(stmt.GetText("format")), + } + n := stmt.GetBytes("public_key", buf) + newSignature.PublicKey = buf[:n:n] + buf = buf[n:] + n = stmt.GetBytes("signature", buf) + newSignature.Signature = buf[:n:n] + + result = append(result, newSignature) + } + if err := stmt.Reset(); err != nil { + return result, fmt.Errorf("signatures: %v", err) + } + return result, nil +} + func recordBuilderStart(conn *sqlite.Conn, buildResultID int64, t time.Time) error { err := sqlitex.ExecuteTransientFS(conn, sqlFiles(), "build/set_builder_start.sql", &sqlitex.ExecOptions{ Named: map[string]any{ @@ -1011,6 +1140,18 @@ func unmarshalJSONString(data string, out any, opts ...jsonv2.Options) error { return jsonv2.UnmarshalRead(strings.NewReader(data), out, opts...) } +func unmarshalHash(algo string, bits []byte) (nix.Hash, error) { + t, err := nix.ParseHashType(algo) + if err != nil { + return nix.Hash{}, err + } + if got, want := len(bits), t.Size(); got != want { + return nix.Hash{}, fmt.Errorf("unmarshal hash: digest is incorrect size (%d instead of %d) for %s", + got, want, t) + } + return nix.NewHash(t, bits), nil +} + // readonlySavepoint starts a new SAVEPOINT. // The caller is responsible for calling endFn // to roll back the SAVEPOINT and remove it from the transaction stack. diff --git a/internal/backend/realize.go b/internal/backend/realize.go index b67f335..edeedec 100644 --- a/internal/backend/realize.go +++ b/internal/backend/realize.go @@ -309,22 +309,6 @@ func (s *Server) newBuilder(id uuid.UUID, derivations map[zbstore.Path]*zbstore. } } -func (b *builder) makeBuildResult(drvPath zbstore.Path) *zbstorerpc.BuildResult { - result := new(zbstorerpc.BuildResult) - for _, outputName := range xmaps.SortedKeys(b.derivations[drvPath].Outputs) { - var p zbstorerpc.Nullable[zbstore.Path] - p.X, p.Valid = b.lookup(zbstore.OutputReference{ - DrvPath: drvPath, - OutputName: outputName, - }) - result.Outputs = append(result.Outputs, &zbstorerpc.RealizeOutput{ - Name: outputName, - Path: p, - }) - } - return result -} - func (b *builder) toEquivalenceClass(ref zbstore.OutputReference) (_ equivalenceClass, ok bool) { if ref.OutputName == "" { return equivalenceClass{}, false @@ -725,7 +709,7 @@ func (b *builder) do(ctx context.Context, drvPath zbstore.Path, outputNames sets } defer endFn(&err) - buildResultID, err = insertBuildResult(conn, b.id, drvPath, startTime) + buildResultID, err = insertBuildResult(conn, b.id, drvPath, drvHash, startTime) if err != nil { return fmt.Errorf("build %s: %v", drvPath, err) } @@ -1637,7 +1621,7 @@ func (b *builder) recordRealizations(ctx context.Context, conn *sqlite.Conn, drv } defer endFn(&err) - if err := recordRealizations(ctx, conn, drvHash, outputs); err != nil { + if err := recordRealizations(ctx, conn, b.server.keyring, drvHash, outputs); err != nil { return err } buildOutputs := func(yield func(string, zbstore.Path) bool) { diff --git a/internal/backend/realize_test.go b/internal/backend/realize_test.go index 9e5400f..efa6cfe 100644 --- a/internal/backend/realize_test.go +++ b/internal/backend/realize_test.go @@ -5,6 +5,7 @@ package backend_test import ( "bytes" + "crypto/ed25519" "fmt" "net/http" "net/http/httptest" @@ -16,6 +17,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" . "zb.256lights.llc/pkg/internal/backend" "zb.256lights.llc/pkg/internal/backendtest" "zb.256lights.llc/pkg/internal/jsonrpc" @@ -745,21 +747,16 @@ func TestRealizeFailure(t *testing.T) { }, }, } - buildType := reflect.TypeFor[zbstorerpc.Build]() diff := cmp.Diff( want, got, cmp.FilterPath( func(p cmp.Path) bool { - if p.Index(-2).Type() != buildType { - return false - } - fieldName := p.Last().(cmp.StructField).Name() - return fieldName == "StartedAt" || - fieldName == "EndedAt" + return isFieldAnyOf[zbstorerpc.Build](p, "StartedAt", "EndedAt") }, cmp.Ignore(), ), buildResultOption, + ignoreDrvHashOption, ) if diff != "" { t.Errorf("build (-want +got):\n%s", diff) @@ -863,21 +860,16 @@ func TestRealizeNoOutput(t *testing.T) { }, }, } - buildType := reflect.TypeFor[zbstorerpc.Build]() diff := cmp.Diff( want, got, cmp.FilterPath( func(p cmp.Path) bool { - if p.Index(-2).Type() != buildType { - return false - } - fieldName := p.Last().(cmp.StructField).Name() - return fieldName == "StartedAt" || - fieldName == "EndedAt" + return isFieldAnyOf[zbstorerpc.Build](p, "StartedAt", "EndedAt") }, cmp.Ignore(), ), buildResultOption, + ignoreDrvHashOption, ) if diff != "" { t.Errorf("build (-want +got):\n%s", diff) @@ -1057,14 +1049,180 @@ func TestRealizeFetchURL(t *testing.T) { checkSingleFileOutput(t, drvPath, wantOutputPath, []byte(fileContent), got) } +func TestRealizeSignature(t *testing.T) { + ctx, cancel := testcontext.New(t) + defer cancel() + dir := backendtest.NewStoreDirectory(t) + + testKey := ed25519.PrivateKey{ + 0xf8, 0xd3, 0x03, 0x35, 0xfb, 0xe3, 0x0a, 0x67, + 0x53, 0xf6, 0x62, 0xeb, 0xf7, 0x36, 0x9d, 0x61, + 0x05, 0xf0, 0x17, 0xf9, 0x8f, 0x2e, 0xc4, 0xe8, + 0x33, 0x0d, 0xfa, 0xc9, 0x7e, 0xf0, 0xe8, 0x70, + 0x95, 0x09, 0x22, 0xbd, 0x27, 0x65, 0xac, 0x30, + 0x63, 0xc2, 0x01, 0x3f, 0x54, 0xd9, 0x8f, 0x79, + 0xf4, 0xd1, 0x60, 0x01, 0xf7, 0x62, 0x49, 0x61, + 0x91, 0xbd, 0x66, 0xd7, 0x62, 0x51, 0x94, 0x70, + } + testPublicKey := testKey.Public().(ed25519.PublicKey) + + const inputContent = "Hello, World!\n" + exportBuffer := new(bytes.Buffer) + exporter := zbstore.NewExportWriter(exportBuffer) + inputFilePath, _, err := storetest.ExportSourceFile(exporter, []byte(inputContent), storetest.SourceExportOptions{ + Name: "hello.txt", + Directory: dir, + }) + if err != nil { + t.Fatal(err) + } + const wantOutputName = "hello2.txt" + drvContent := &zbstore.Derivation{ + Name: wantOutputName, + Dir: dir, + System: system.Current().String(), + Env: map[string]string{ + "in": string(inputFilePath), + "out": zbstore.HashPlaceholder("out"), + }, + InputSources: *sets.NewSorted( + inputFilePath, + ), + Outputs: map[string]*zbstore.DerivationOutputType{ + zbstore.DefaultDerivationOutputName: zbstore.RecursiveFileFloatingCAOutput(nix.SHA256), + }, + } + drvContent.Builder, drvContent.Args = catcatBuilder() + drvPath, _, err := storetest.ExportDerivation(exporter, drvContent) + if err != nil { + t.Fatal(err) + } + if err := exporter.Close(); err != nil { + t.Fatal(err) + } + drvHash, err := drvContent.SHA256RealizationHash(func(ref zbstore.OutputReference) (zbstore.Path, bool) { + return "", false + }) + if err != nil { + t.Fatal(err) + } + + _, client, err := backendtest.NewServer(ctx, t, dir, &backendtest.Options{ + TempDir: t.TempDir(), + Options: Options{ + Keyring: &Keyring{ + Ed25519: []ed25519.PrivateKey{testKey}, + }, + }, + }) + if err != nil { + t.Fatal(err) + } + codec, releaseCodec, err := storeCodec(ctx, client) + if err != nil { + t.Fatal(err) + } + err = codec.Export(nil, exportBuffer) + releaseCodec() + if err != nil { + t.Fatal(err) + } + + realizeResponse := new(zbstorerpc.RealizeResponse) + err = jsonrpc.Do(ctx, client, zbstorerpc.RealizeMethod, realizeResponse, &zbstorerpc.RealizeRequest{ + DrvPaths: []zbstore.Path{drvPath}, + }) + if err != nil { + t.Fatal("RPC error:", err) + } + if realizeResponse.BuildID == "" { + t.Fatal("no build ID returned") + } + + got, err := backendtest.WaitForSuccessfulBuild(ctx, client, realizeResponse.BuildID) + if err != nil { + gotLog, _ := backendtest.ReadLog(ctx, client, realizeResponse.BuildID, drvPath) + t.Fatalf("build drv: %v\nlog:\n%s", err, gotLog) + } + + const wantOutputContent = "Hello, World!\nHello, World!\n" + wantOutputPath, err := singleFileOutputPath(dir, wantOutputName, []byte(wantOutputContent), zbstore.References{}) + if err != nil { + t.Fatal(err) + } + + gotResult, err := got.ResultForPath(drvPath) + if err != nil { + t.Error(err) + } + if gotResult == nil { + return + } + + outputRef := zbstore.RealizationOutputReference{ + DerivationHash: drvHash, + OutputName: "out", + } + realization := &zbstore.Realization{ + OutputPath: wantOutputPath, + Signatures: []*zbstore.RealizationSignature{ + { + Format: zbstore.Ed25519SignatureFormat, + PublicKey: testPublicKey, + }, + }, + } + sig, err := zbstore.SignRealizationWithEd25519(outputRef, realization, testKey) + if err != nil { + t.Fatal(err) + } + want := &zbstorerpc.BuildResult{ + DrvPath: drvPath, + DrvHash: zbstorerpc.NonNull(drvHash), + Status: zbstorerpc.BuildSuccess, + Outputs: []*zbstorerpc.RealizeOutput{ + { + Name: zbstore.DefaultDerivationOutputName, + Path: zbstorerpc.NonNull(wantOutputPath), + Signatures: []*zbstore.RealizationSignature{sig}, + }, + }, + } + diff := cmp.Diff( + want, gotResult, + buildResultOption, + ) + if diff != "" { + t.Errorf("realize response (-want +got):\n%s", diff) + } +} + var buildResultOption = cmp.Options{ cmp.FilterPath(func(p cmp.Path) bool { - if p.Index(-2).Type() != reflect.TypeFor[zbstorerpc.BuildResult]() { - return false - } - name := p.Last().(cmp.StructField).Name() - return name == "LogSize" + return isFieldAnyOf[zbstorerpc.BuildResult](p, "LogSize") }, cmp.Ignore()), + cmp.FilterPath(isRealizeOutputSignaturesField, cmpopts.EquateEmpty()), +} + +var ignoreDrvHashOption = cmp.FilterPath(func(p cmp.Path) bool { + return isFieldAnyOf[zbstorerpc.BuildResult](p, "DrvHash") +}, cmp.Ignore()) + +func isRealizeOutputSignaturesField(p cmp.Path) bool { + return isFieldAnyOf[zbstorerpc.RealizeOutput](p, "Signatures") +} + +func isFieldAnyOf[T any](p cmp.Path, names ...string) bool { + if p.Index(-2).Type() != reflect.TypeFor[T]() { + return false + } + name := p.Last().(cmp.StructField).Name() + for _, n := range names { + if name == n { + return true + } + } + return false } func checkSingleFileOutput(tb testing.TB, drvPath, wantOutputPath zbstore.Path, wantOutputContent []byte, resp *zbstorerpc.Build) { @@ -1088,7 +1246,12 @@ func checkSingleFileOutput(tb testing.TB, drvPath, wantOutputPath zbstore.Path, }, }, } - diff := cmp.Diff(want, got, buildResultOption) + diff := cmp.Diff( + want, got, + buildResultOption, + ignoreDrvHashOption, + cmp.FilterPath(isRealizeOutputSignaturesField, cmp.Ignore()), + ) if diff != "" { tb.Errorf("realize response (-want +got):\n%s", diff) } diff --git a/internal/backend/sql/build/insert_result.sql b/internal/backend/sql/build/insert_result.sql index 4592911..9015fcc 100644 --- a/internal/backend/sql/build/insert_result.sql +++ b/internal/backend/sql/build/insert_result.sql @@ -1,9 +1,13 @@ insert into "build_results" ( "build_id", "drv_path", + "drv_hash", "started_at" ) values ( (select "id" from "builds" where "uuid" = uuid(:build_id)), (select "id" from "paths" where "path" = :drv_path), + (select "id" from "drv_hashes" + where "algorithm" = :drv_hash_algorithm + and "bits" = :drv_hash_bits), :timestamp_millis ) returning "id"; diff --git a/internal/backend/sql/build/result_signatures.sql b/internal/backend/sql/build/result_signatures.sql new file mode 100644 index 0000000..de3376c --- /dev/null +++ b/internal/backend/sql/build/result_signatures.sql @@ -0,0 +1,20 @@ +select + "signature_public_keys"."format" as "format", + "signature_public_keys"."public_key" as "public_key", + "signatures"."signature" as "signature" +from + "signatures" + join "signature_public_keys" on "signature_public_keys"."id" = "signatures"."public_key_id" + join "build_results" on "build_results"."drv_hash" = "signatures"."drv_hash" + join "paths" as "drv_path" on "drv_path"."id" = "build_results"."drv_path" + join "paths" as "output_path" on "output_path"."id" = "signatures"."output_path" + join "builds" on "builds"."id" = "build_results"."build_id" +where + "builds"."uuid" = uuid(:build_id) and + "drv_path"."path" = :drv_path and + "signatures"."output_name" = :output_name and + "output_path"."path" = :output_path +order by + "signature_public_keys"."format", + "signature_public_keys"."public_key", + "signatures"."signature"; diff --git a/internal/backend/sql/build/results.sql b/internal/backend/sql/build/results.sql index 104ce94..e747a73 100644 --- a/internal/backend/sql/build/results.sql +++ b/internal/backend/sql/build/results.sql @@ -1,5 +1,7 @@ select "drv_path"."path" as "drv_path", + "drv_hash"."algorithm" as "drv_hash_algorithm", + "drv_hash"."bits" as "drv_hash_bits", "build_results"."status" as "status", "build_results"."started_at" as "started_at", "build_results"."ended_at" as "ended_at", @@ -11,6 +13,7 @@ from "build_results" join "builds" on "builds"."id" = "build_results"."build_id" join "paths" as "drv_path" on "drv_path"."id" = "build_results"."drv_path" + left join "drv_hashes" as "drv_hash" on "drv_hash"."id" = "build_results"."drv_hash" left join "build_outputs" as "outputs" on "outputs"."result_id" = "build_results"."id" left join "paths" as "output_path" on "output_path"."id" = "outputs"."output_path" where diff --git a/internal/backend/sql/insert_public_key.sql b/internal/backend/sql/insert_public_key.sql new file mode 100644 index 0000000..727cf36 --- /dev/null +++ b/internal/backend/sql/insert_public_key.sql @@ -0,0 +1,3 @@ +insert into "signature_public_keys" ("format", "public_key") +values (:format, :public_key) +on conflict ("format", "public_key") do nothing; diff --git a/internal/backend/sql/insert_signature.sql b/internal/backend/sql/insert_signature.sql new file mode 100644 index 0000000..029b7d7 --- /dev/null +++ b/internal/backend/sql/insert_signature.sql @@ -0,0 +1,13 @@ +insert into "signatures" ( + "drv_hash", + "output_name", + "output_path", + "public_key_id", + "signature" +) values ( + (select "id" from "drv_hashes" where ("algorithm", "bits") = (:drv_hash_algorithm, :drv_hash_bits)), + :output_name, + (select "id" from "paths" where "path" = :output_path), + (select "id" from "signature_public_keys" where ("format", "public_key") = (:format, :public_key)), + :signature +); diff --git a/internal/backend/sql/schema/04.sql b/internal/backend/sql/schema/04.sql new file mode 100644 index 0000000..fef7c04 --- /dev/null +++ b/internal/backend/sql/schema/04.sql @@ -0,0 +1,24 @@ +create table "signature_public_keys" ( + "id" integer primary key not null, + "format" text not null, + "public_key" blob not null, + + unique ("format", "public_key") +); + +create table "signatures" ( + "id" integer primary key not null, + + "drv_hash" integer not null, + "output_name" text not null, + "output_path" integer not null, + + "public_key_id" integer + references "signature_public_keys", + "signature" blob, + + foreign key ("drv_hash", "output_name", "output_path") references "realizations" + on delete cascade +); + +alter table "build_results" add column "drv_hash" integer references "drv_hashes"; diff --git a/internal/zbstorerpc/zbstorerpc.go b/internal/zbstorerpc/zbstorerpc.go index 03db79b..33f38f7 100644 --- a/internal/zbstorerpc/zbstorerpc.go +++ b/internal/zbstorerpc/zbstorerpc.go @@ -207,10 +207,11 @@ type GetBuildResultRequest struct { // BuildResult is the result of a single derivation in a [Build]. type BuildResult struct { - DrvPath zbstore.Path `json:"drvPath"` - Status BuildStatus `json:"status"` - Outputs []*RealizeOutput `json:"outputs"` - LogSize int64 `json:"logSize"` + DrvPath zbstore.Path `json:"drvPath"` + DrvHash Nullable[nix.Hash] `json:"drvHash"` + Status BuildStatus `json:"status"` + Outputs []*RealizeOutput `json:"outputs"` + LogSize int64 `json:"logSize"` } // OutputForName returns the [*RealizeOutput] with the given name. @@ -267,6 +268,8 @@ type RealizeOutput struct { // Path is the store path of the output if successfully built, // or null if the build failed. Path Nullable[zbstore.Path] `json:"path"` + // Signatures is the set of signatures for the realization. + Signatures []*zbstore.RealizationSignature `json:"signatures"` } // CancelBuildMethod is the name of the method that informs the store