Skip to content

frontend: implement readFile function #149

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zombiezen merged 2 commits into from
Jun 23, 2025
Merged

frontend: implement readFile function #149

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bbe2f30
frontend: implement readFile function
winterqt Jun 19, 2025
5a55bc4
Apply suggestions from code review
winterqt Jun 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions internal/frontend/eval.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,7 @@ func (eval *Eval) initZygote() error {
"import": eval.importFunction,
"toFile": eval.toFileFunction,
"path": eval.pathFunction,
"readFile": eval.readFileFunction,
"storePath": eval.storePathFunction,
}
if err := lua.SetPureFunctions(ctx, l, 0, extraBaseFunctions); err != nil {
Expand Down
53 changes: 1 addition & 52 deletions internal/frontend/module.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,13 @@ package frontend

import (
"context"
"fmt"
"iter"
"slices"
"strings"
"sync"

"zb.256lights.llc/pkg/internal/lua"
"zb.256lights.llc/pkg/internal/luacode"
"zb.256lights.llc/pkg/internal/zbstorerpc"
"zb.256lights.llc/pkg/sets"
"zb.256lights.llc/pkg/zbstore"
)

const moduleTypeName = "module"
Expand Down Expand Up @@ -74,54 +70,7 @@ func (eval *Eval) importFunction(ctx context.Context, l *lua.State) (int, error)
}
filenameContext := l.StringContext(1)

// TODO(someday): If we have dependencies and we're using a non-local store,
// export the store object and read it.
toRealize := make(sets.Set[zbstore.OutputReference])
placeholders := make(map[string]zbstore.OutputReference)
for dep := range filenameContext {
c, err := parseContextString(dep)
if err != nil {
l.PushNil()
l.PushString(fmt.Sprintf("internal error: %v", err))
return 2, nil
}
if c.outputReference.IsZero() {
continue
}
placeholder := zbstore.UnknownCAOutputPlaceholder(c.outputReference)
if !strings.Contains(filename, placeholder) {
continue
}
toRealize.Add(c.outputReference)
placeholders[placeholder] = c.outputReference
}
if toRealize.Len() > 0 {
results, err := eval.store.Realize(ctx, toRealize)
if err != nil {
l.PushNil()
l.PushString(err.Error())
return 2, nil
}

var rewrites []string
for placeholder, outputReference := range placeholders {
outputPath, err := zbstorerpc.FindRealizeOutput(slices.Values(results), outputReference)
if err != nil {
l.PushNil()
l.PushString(err.Error())
return 2, nil
}
if !outputPath.Valid || outputPath.X == "" {
l.PushNil()
l.PushString(fmt.Sprintf("realize %v: build failed", outputReference))
return 2, nil
}
rewrites = append(rewrites, placeholder, string(outputPath.X))
}
filename = strings.NewReplacer(rewrites...).Replace(filename)
}

filename, err = absSourcePath(l, eval.storeDir, filename, filenameContext)
filename, err = absSourcePathWithDeps(ctx, l, eval, filename, filenameContext)
if err != nil {
l.PushNil()
l.PushString(err.Error())
Expand Down
70 changes: 70 additions & 0 deletions internal/frontend/path_eval.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,13 @@ import (
"io/fs"
"os"
"path/filepath"
"slices"
"strings"
"sync"

"zb.256lights.llc/pkg/internal/lua"
"zb.256lights.llc/pkg/internal/osutil"
"zb.256lights.llc/pkg/internal/zbstorerpc"
"zb.256lights.llc/pkg/sets"
"zb.256lights.llc/pkg/zbstore"
"zombiezen.com/go/log"
Expand Down Expand Up @@ -246,6 +249,27 @@ func (eval *Eval) pathFunction(ctx context.Context, l *lua.State) (nResults int,
return 1, nil
}

func (eval *Eval) readFileFunction(ctx context.Context, l *lua.State) (int, error) {
path, err := lua.CheckString(l, 1)
if err != nil {
return 0, err
}
pcontext := l.StringContext(1)

absPath, err := absSourcePathWithDeps(ctx, l, eval, path, pcontext)
if err != nil {
return 0, fmt.Errorf("readFile: %v", err)
}

content, err := osutil.ReadFileString(absPath)
if err != nil {
return 0, fmt.Errorf("readFile: reading file: %v", err)
}

l.PushString(content)
return 1, nil
}

func (eval *Eval) toFileFunction(ctx context.Context, l *lua.State) (int, error) {
name, err := lua.CheckString(l, 1)
if err != nil {
Expand Down Expand Up @@ -394,6 +418,52 @@ func absSourcePath(l *lua.State, dir zbstore.Directory, path string, context set
return path, nil
}

// absSourcePathWithDeps takes a source path passed as an argument from Lua to Go
// and resolves it relative to the calling function, taking into account
// any dependencies the string may have.
func absSourcePathWithDeps(ctx context.Context, l *lua.State, eval *Eval, filename string, filenameContext sets.Set[string]) (path string, err error) {
// TODO(someday): If we have dependencies and we're using a non-local store,
// export the store object and read it.
toRealize := make(sets.Set[zbstore.OutputReference])
placeholders := make(map[string]zbstore.OutputReference)
for dep := range filenameContext {
c, err := parseContextString(dep)
if err != nil {
return "", fmt.Errorf("internal error: %w", err)
}
if c.outputReference.IsZero() {
continue
}
placeholder := zbstore.UnknownCAOutputPlaceholder(c.outputReference)
if !strings.Contains(filename, placeholder) {
continue
}
toRealize.Add(c.outputReference)
placeholders[placeholder] = c.outputReference
}
if toRealize.Len() > 0 {
results, err := eval.store.Realize(ctx, toRealize)
if err != nil {
return "", err
}

var rewrites []string
for placeholder, outputReference := range placeholders {
outputPath, err := zbstorerpc.FindRealizeOutput(slices.Values(results), outputReference)
if err != nil {
return "", err
}
if !outputPath.Valid || outputPath.X == "" {
return "", fmt.Errorf("realize %v: build failed", outputReference)
}
rewrites = append(rewrites, placeholder, string(outputPath.X))
}
filename = strings.NewReplacer(rewrites...).Replace(filename)
}

return absSourcePath(l, eval.storeDir, filename, filenameContext)
}

func pathInStore(path string, dir zbstore.Directory) bool {
return strings.HasPrefix(path, string(dir)) &&
(len(path) <= len(dir) || path[len(dir)] == byte(filepath.Separator))
Expand Down
44 changes: 44 additions & 0 deletions internal/frontend/path_eval_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,50 @@ func TestPath(t *testing.T) {
})
}

func TestReadFile(t *testing.T) {
wantContent, err := os.ReadFile(filepath.Join("testdata", "hello.txt"))
if err != nil {
t.Fatal(err)
}

ctx, cancel := testcontext.New(t)
defer cancel()
storeDir := backendtest.NewStoreDirectory(t)

_, store, err := backendtest.NewServer(ctx, t, storeDir, &backendtest.Options{
TempDir: t.TempDir(),
})
if err != nil {
t.Fatal(err)
}
testStore := newTestRPCStore(store)
eval, err := NewEval(&Options{
Store: testStore,
StoreDirectory: storeDir,
})
if err != nil {
t.Fatal(err)
}
defer func() {
if err := eval.Close(); err != nil {
t.Error("eval.Close:", err)
}
}()

got, err := eval.Expression(ctx, `readFile("testdata/hello.txt")`)
if err != nil {
t.Fatal(err)
}
gotString, ok := got.(string)
if !ok {
t.Fatalf("expression result is %T; want string", got)
}

if !bytes.Equal([]byte(gotString), wantContent) {
t.Errorf("gotString = %q; want %q", gotString, wantContent)
}
}

// compareDirectoryToTestdata compares dir to the directory at testdata/dir.
// If dir does not contain exactly the files named in wantFiles,
// then compareDirectoryToTestdata logs a failure to tb.
Expand Down
17 changes: 17 additions & 0 deletions internal/osutil/osutil.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@ package osutil
import (
"errors"
"fmt"
"io"
"iter"
"os"
"path/filepath"
"runtime"
"strings"
"syscall"
"time"
)
Expand Down Expand Up @@ -204,3 +206,18 @@ func MkdirAllInRoot(root *os.Root, path string, perm os.FileMode) error {
func UnmountAndRemoveAll(path string) error {
return removeAll(path)
}

// ReadFileString reads the entire content of name into a string.
func ReadFileString(name string) (string, error) {
f, err := os.Open(name)
if err != nil {
return "", err
}
defer f.Close()
sb := new(strings.Builder)
if info, err := f.Stat(); err == nil {
sb.Grow(int(info.Size()))
}
_, err = io.Copy(sb, f)
return sb.String(), err
}
29 changes: 7 additions & 22 deletions zbstore/ca_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,37 +4,36 @@
package zbstore

import (
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"testing"

"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"zb.256lights.llc/pkg/internal/osutil"
"zombiezen.com/go/nix"
"zombiezen.com/go/nix/nar"
)

func TestSourceSHA256ContentAddress(t *testing.T) {
machoSelfReferenceNAR, err := readFileString(filepath.Join("testdata", "macho-selfref-aarch64.nar"))
machoSelfReferenceNAR, err := osutil.ReadFileString(filepath.Join("testdata", "macho-selfref-aarch64.nar"))
if err != nil {
t.Fatal(err)
}
machoZeroedNAR, err := readFileString(filepath.Join("testdata", "macho-zeroed-aarch64.nar"))
machoZeroedNAR, err := osutil.ReadFileString(filepath.Join("testdata", "macho-zeroed-aarch64.nar"))
if err != nil {
t.Fatal(err)
}
machoUniversalSelfReferenceNAR, err := readFileString(filepath.Join("testdata", "macho-selfref-universal.nar"))
machoUniversalSelfReferenceNAR, err := osutil.ReadFileString(filepath.Join("testdata", "macho-selfref-universal.nar"))
if err != nil {
t.Fatal(err)
}
machoUniversalZeroedNAR, err := readFileString(filepath.Join("testdata", "macho-zeroed-universal.nar"))
machoUniversalZeroedNAR, err := osutil.ReadFileString(filepath.Join("testdata", "macho-zeroed-universal.nar"))
if err != nil {
t.Fatal(err)
}
machoNoRefsNAR, err := readFileString(filepath.Join("testdata", "macho-norefs-aarch64.nar"))
machoNoRefsNAR, err := osutil.ReadFileString(filepath.Join("testdata", "macho-norefs-aarch64.nar"))
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -346,7 +345,7 @@ func TestSourceSHA256ContentAddress(t *testing.T) {
}

func BenchmarkSourceSHA256ContentAddress(b *testing.B) {
machoUniversalSelfReferenceNAR, err := readFileString(filepath.Join("testdata", "macho-selfref-universal.nar"))
machoUniversalSelfReferenceNAR, err := osutil.ReadFileString(filepath.Join("testdata", "macho-selfref-universal.nar"))
if err != nil {
b.Fatal(err)
}
Expand Down Expand Up @@ -385,17 +384,3 @@ func BenchmarkSourceSHA256ContentAddress(b *testing.B) {
}
})
}

func readFileString(name string) (string, error) {
f, err := os.Open(name)
if err != nil {
return "", err
}
defer f.Close()
sb := new(strings.Builder)
if info, err := f.Stat(); err == nil {
sb.Grow(int(info.Size()))
}
_, err = io.Copy(sb, f)
return sb.String(), err
}
13 changes: 7 additions & 6 deletions zbstore/rewrite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,31 +10,32 @@ import (

"github.com/google/go-cmp/cmp"
"zb.256lights.llc/pkg/bytebuffer"
"zb.256lights.llc/pkg/internal/osutil"
"zombiezen.com/go/nix"
)

func TestRewrite(t *testing.T) {
machoSelfReferenceNAR, err := readFileString(filepath.Join("testdata", "macho-selfref-aarch64.nar"))
machoSelfReferenceNAR, err := osutil.ReadFileString(filepath.Join("testdata", "macho-selfref-aarch64.nar"))
if err != nil {
t.Fatal(err)
}
machoZeroedNAR, err := readFileString(filepath.Join("testdata", "macho-zeroed-aarch64.nar"))
machoZeroedNAR, err := osutil.ReadFileString(filepath.Join("testdata", "macho-zeroed-aarch64.nar"))
if err != nil {
t.Fatal(err)
}
machoRewrittenNAR, err := readFileString(filepath.Join("testdata", "macho-rewritten-aarch64.nar"))
machoRewrittenNAR, err := osutil.ReadFileString(filepath.Join("testdata", "macho-rewritten-aarch64.nar"))
if err != nil {
t.Fatal(err)
}
machoUniversalSelfReferenceNAR, err := readFileString(filepath.Join("testdata", "macho-selfref-universal.nar"))
machoUniversalSelfReferenceNAR, err := osutil.ReadFileString(filepath.Join("testdata", "macho-selfref-universal.nar"))
if err != nil {
t.Fatal(err)
}
machoUniversalZeroedNAR, err := readFileString(filepath.Join("testdata", "macho-zeroed-universal.nar"))
machoUniversalZeroedNAR, err := osutil.ReadFileString(filepath.Join("testdata", "macho-zeroed-universal.nar"))
if err != nil {
t.Fatal(err)
}
machoUniversalRewrittenNAR, err := readFileString(filepath.Join("testdata", "macho-rewritten-universal.nar"))
machoUniversalRewrittenNAR, err := osutil.ReadFileString(filepath.Join("testdata", "macho-rewritten-universal.nar"))
if err != nil {
t.Fatal(err)
}
Expand Down
Loading