From dd3951a78e84cab51b8c868c635e5cd9f3c3ad1b Mon Sep 17 00:00:00 2001 From: Emil Valeev Date: Thu, 16 Nov 2023 00:40:04 +0600 Subject: [PATCH] wip(builder): merge std mod with entry mod, add lots of todos for multimod flow --- .vscode/launch.json | 31 ++-- cmd/interpreter/main.go | 5 +- .../naive/main.neva.fixme} | 15 +- .../with_error_handling/main.neva} | 16 +-- .../with_subcomponents/main.neva.todo} | 8 +- .../main.neva} | 0 examples/{001_echo.neva => echo/main.neva} | 6 +- .../main.neva} | 9 +- examples/neva.yml | 1 + internal/builder/builder.go | 55 +++++--- internal/builder/builder_test.go | 133 ------------------ internal/compiler/analyzer/analyzer.go | 5 +- internal/compiler/compiler.go | 16 ++- internal/compiler/sourcecode/scope.go | 21 ++- internal/compiler/sourcecode/src.go | 2 +- internal/interpreter/interpreter.go | 6 +- 16 files changed, 121 insertions(+), 208 deletions(-) rename examples/{005_add_two_numbers_1.neva => add_numbers/naive/main.neva.fixme} (70%) rename examples/{006_add_two_numbers_2.neva => add_numbers/with_error_handling/main.neva} (61%) rename examples/{007_add_two_numbers_3.neva => add_numbers/with_subcomponents/main.neva.todo} (87%) rename examples/{000_do_nothing.neva => do_nothing/main.neva} (100%) rename examples/{001_echo.neva => echo/main.neva} (80%) rename examples/{002_hello_world_1.neva => hello_world/main.neva} (52%) create mode 100644 examples/neva.yml delete mode 100644 internal/builder/builder_test.go diff --git a/.vscode/launch.json b/.vscode/launch.json index e4863cb8..5aeec071 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -14,18 +14,6 @@ "outFiles": ["${workspaceFolder}/web/out/**/*.js"], "preLaunchTask": "${defaultBuildTask}" }, - // test extension - // { - // "name": "VSCode Extension Tests", - // "type": "extensionHost", - // "request": "launch", - // "args": [ - // "--extensionDevelopmentPath=${workspaceFolder}", - // "--extensionTestsPath=${workspaceFolder}/out/test/suite/index" - // ], - // "outFiles": ["${workspaceFolder}/out/test/**/*.js"], - // "preLaunchTask": "${defaultBuildTask}" - // }, // run lsp in debugger { "name": "Language server (LSP) ", @@ -36,6 +24,25 @@ "cwd": "${workspaceFolder}", "args": ["-debug"] }, + // === Interpreter === + { + "name": "Interpreter: do_nothing", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/interpreter", + "cwd": "${workspaceFolder}/examples", + "args": ["do_nothing"], + }, + { + "name": "Interpreter: add_numbers", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/interpreter", + "cwd": "${workspaceFolder}/examples", + "args": ["add_numbers/with_error_handling"], + }, // === Other === { "name": "antlr_000_empty.neva", diff --git a/cmd/interpreter/main.go b/cmd/interpreter/main.go index 430d6933..022eca2b 100644 --- a/cmd/interpreter/main.go +++ b/cmd/interpreter/main.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "os" - "path/filepath" "github.com/nevalang/neva/internal/builder" "github.com/nevalang/neva/internal/compiler" @@ -68,13 +67,13 @@ func main() { ), ) - path, err := filepath.Abs(os.Args[1]) + wd, err := os.Getwd() if err != nil { fmt.Println(err) return } - code, err := intr.Interpret(context.Background(), path) + code, err := intr.Interpret(context.Background(), wd, os.Args[1]) if err != nil { fmt.Println(err) return diff --git a/examples/005_add_two_numbers_1.neva b/examples/add_numbers/naive/main.neva.fixme similarity index 70% rename from examples/005_add_two_numbers_1.neva rename to examples/add_numbers/naive/main.neva.fixme index e355cd55..a0861eb1 100644 --- a/examples/005_add_two_numbers_1.neva +++ b/examples/add_numbers/naive/main.neva.fixme @@ -3,21 +3,22 @@ // Program that adds two numbers typed by user and prints the result. // The algorithm is this: read first input, parse it, -// then read the second input and parse too. Add numbers and print the result. +// then read the second input and parse it too. +// Add numbers and print the result. use { - std/tmp + std/core } components { Main(enter) (exit) { nodes { - readFirst tmp.Read - parseFirst tmp.ParseInt - readSecond tmp.Read - parseSecond tmp.ParseInt + readFirst core.Read + parseFirst core.ParseInt + readSecond core.Read + parseSecond core.ParseInt add Add - print tmp.Print + print core.Print } net { in.enter -> readFirst.sig diff --git a/examples/006_add_two_numbers_2.neva b/examples/add_numbers/with_error_handling/main.neva similarity index 61% rename from examples/006_add_two_numbers_2.neva rename to examples/add_numbers/with_error_handling/main.neva index 1a8e52c8..ab7f1aa4 100644 --- a/examples/006_add_two_numbers_2.neva +++ b/examples/add_numbers/with_error_handling/main.neva @@ -1,20 +1,20 @@ +// Handle errors + use { - std/tmp - github/nevalang/neva/pkg/typesystem - some/really/deeply/nested/path/to/local/package/inthe/the/project + std/core } components { Main(enter) (exit) { nodes { - readFirst tmp.Read - readSecond tmp.Read + readFirst core.Read + readSecond core.Read - parseFirst tmp.ParseInt - parseSecond tmp.ParseInt + parseFirst core.ParseInt + parseSecond core.ParseInt add Add - print tmp.Print + print core.Print } net { diff --git a/examples/007_add_two_numbers_3.neva b/examples/add_numbers/with_subcomponents/main.neva.todo similarity index 87% rename from examples/007_add_two_numbers_3.neva rename to examples/add_numbers/with_subcomponents/main.neva.todo index f4d0f638..cb8f4163 100644 --- a/examples/007_add_two_numbers_3.neva +++ b/examples/add_numbers/with_subcomponents/main.neva.todo @@ -1,7 +1,7 @@ // Our program turns out to be quite big! Let's split it into sub-components? use { - std/tmp + std/core } components { @@ -10,7 +10,7 @@ components { readFirstInt ReadInt readSecondInt ReadInt add Add - print tmp.Print + print core.Print } net { in.enter -> readFirstInt.sig @@ -28,8 +28,8 @@ components { ReadInt(sig) (v int, err str) { nodes { - read tmp.Read - parse tmp.ParseInt + read core.Read + parse core.ParseInt } net { in.sig -> read.sig diff --git a/examples/000_do_nothing.neva b/examples/do_nothing/main.neva similarity index 100% rename from examples/000_do_nothing.neva rename to examples/do_nothing/main.neva diff --git a/examples/001_echo.neva b/examples/echo/main.neva similarity index 80% rename from examples/001_echo.neva rename to examples/echo/main.neva index db2198d0..dbae1d17 100644 --- a/examples/001_echo.neva +++ b/examples/echo/main.neva @@ -2,14 +2,14 @@ // And it does so in an infinite loop. use { - std/tmp + std/core } components { Main(enter) (exit) { nodes { - read tmp.Read - print tmp.Print + read core.Read + print core.Print } net { in.enter -> read.sig diff --git a/examples/002_hello_world_1.neva b/examples/hello_world/main.neva similarity index 52% rename from examples/002_hello_world_1.neva rename to examples/hello_world/main.neva index 52c2ef4f..66c98ebb 100644 --- a/examples/002_hello_world_1.neva +++ b/examples/hello_world/main.neva @@ -1,5 +1,5 @@ use { - std/tmp + std/core } const { @@ -9,11 +9,12 @@ const { components { Main(enter) (exit) { nodes { - print tmp.Print + print core.Print } net { - enter -> out.exit - msg -> out.exit + in.enter -> out.exit + msg -> print.v + print.v -> out.exit } } } \ No newline at end of file diff --git a/examples/neva.yml b/examples/neva.yml new file mode 100644 index 00000000..c14c3e92 --- /dev/null +++ b/examples/neva.yml @@ -0,0 +1 @@ +compiler: 0.0.1 \ No newline at end of file diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 2ab84260..3c6bf664 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -12,6 +12,7 @@ import ( "github.com/nevalang/neva/internal/compiler" "github.com/nevalang/neva/internal/compiler/parser" + "github.com/nevalang/neva/internal/compiler/sourcecode" ) type Builder struct { @@ -21,20 +22,27 @@ type Builder struct { } func (b Builder) Build(ctx context.Context, workdir string) (compiler.Build, error) { - // build user module mods := map[string]compiler.RawModule{} + + // this supposed to work recursively but currently only root module is processed correctly entryMod, err := b.buildModule(ctx, workdir, mods) if err != nil { return compiler.Build{}, fmt.Errorf("build entry mod: %w", err) } mods["entry"] = entryMod - // build stdlib module + // TODO in the future we not going to merge std module into root one + // because that won't work with normal multi-module flow + // but currently we do this because it works for single-mod flow (which we only support for the moment) stdMod, err := b.buildModule(ctx, b.stdlibLocation, mods) if err != nil { return compiler.Build{}, fmt.Errorf("build entry mod: %w", err) } - mods["std"] = stdMod + + // merge std module with the entry module (TODO remove this) + for name, stdPkg := range stdMod.Packages { + mods["entry"].Packages["std/"+name] = stdPkg + } return compiler.Build{ EntryModule: "entry", @@ -57,20 +65,11 @@ func (b Builder) buildModule( return compiler.RawModule{}, fmt.Errorf("parse manifest: %w", err) } - // process module deps (download if needed and build) + // FIXME this isn't tested at all (multi-module flow isn't implemented yet) for name, dep := range manifest.Deps { - depPath := fmt.Sprintf("%s/%s_%s", b.thirdPartyLocation, dep.Addr, dep.Version) - if _, err := os.Stat(depPath); err != nil { // check if directory with this dependency exists - if os.IsNotExist(err) { // if not, clone the repo - if _, err := git.PlainClone(depPath, false, &git.CloneOptions{ - URL: fmt.Sprintf("https://%s", dep.Addr), - ReferenceName: plumbing.NewTagReferenceName(dep.Version), - }); err != nil { - return compiler.RawModule{}, err - } - } else { // if it's an unknown error then return - return compiler.RawModule{}, fmt.Errorf("os stat: %w", err) - } + depPath, err := b.downloadDep(dep) + if err != nil { + return compiler.RawModule{}, fmt.Errorf("%w", err) } rawMod, err := b.buildModule(ctx, depPath, mods) @@ -93,6 +92,23 @@ func (b Builder) buildModule( }, nil } +func (b Builder) downloadDep(dep sourcecode.Dependency) (string, error) { + depPath := fmt.Sprintf("%s/%s_%s", b.thirdPartyLocation, dep.Addr, dep.Version) + if _, err := os.Stat(depPath); err != nil { + if os.IsNotExist(err) { + if _, err := git.PlainClone(depPath, false, &git.CloneOptions{ + URL: fmt.Sprintf("https://%s", dep.Addr), + ReferenceName: plumbing.NewTagReferenceName(dep.Version), + }); err != nil { + return "", err + } + } else { + return "", fmt.Errorf("os stat: %w", err) + } + } + return depPath, nil +} + func readManifestYaml(workdir string) ([]byte, error) { rawManifest, err := os.ReadFile(workdir + "/neva.yml") if err == nil { @@ -107,7 +123,6 @@ func readManifestYaml(workdir string) ([]byte, error) { return rawManifest, nil } -// walk recursively traverses the directory assuming that is a neva module (set of packages with source code files). func walk(rootPath string, prog map[string]compiler.RawPackage) error { if err := filepath.Walk(rootPath, func(filePath string, info os.FileInfo, err error) error { if err != nil { @@ -149,10 +164,10 @@ func getPkgName(rootPath, filePath string) string { return strings.TrimPrefix(dirPath, rootPath+"/") } -func MustNew(stdPkgPath, thirdPartyLocation string, parser parser.Parser) Builder { +func MustNew(stdlibPath, thirdpartyPath string, parser parser.Parser) Builder { return Builder{ - stdlibLocation: stdPkgPath, - thirdPartyLocation: thirdPartyLocation, + stdlibLocation: stdlibPath, + thirdPartyLocation: thirdpartyPath, parser: parser, } } diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go deleted file mode 100644 index 8d936ab0..00000000 --- a/internal/builder/builder_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package builder_test - -import ( - "context" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/require" - "golang.org/x/exp/maps" - "gopkg.in/yaml.v3" - - "github.com/nevalang/neva/internal/builder" - "github.com/nevalang/neva/internal/compiler" - "github.com/nevalang/neva/internal/compiler/parser" - src "github.com/nevalang/neva/internal/compiler/sourcecode" -) - -func Test_Build(t *testing.T) { - // === SETUP === - wd, err := os.Getwd() - require.NoError(t, err) - tmpPath := filepath.Join(wd, "tmp") - - manifest := src.Manifest{ - Compiler: "0.0.1", - Deps: map[string]src.Dependency{ - "github.com/nevalang/x": { - Addr: "github.com/nevalang/x", // this repo must be available throughout the network - Version: "0.0.1", // this tag must exist in the repo - }, - }, - } - - files := map[string][]string{ - "foo": {"1.neva", "2.neva"}, - "foo/bar": {"3.neva"}, - "baz": {"4.neva"}, - } - - err = createMod(manifest, files, tmpPath) - require.NoError(t, err) - - // === TEARDOWN === - t.Cleanup(func() { - if err := os.RemoveAll(tmpPath); err != nil { - t.Fatal(err) - } - }) - - // === TEST === - builder := builder.MustNew( - "/Users/emil/projects/neva/std", - "/Users/emil/projects/neva/thirdparty", - parser.MustNew(false), - ) - - actualBuild, err := builder.Build(context.Background(), tmpPath) - require.NoError(t, err) - - expectedBuild := compiler.Build{ - EntryModule: "entry", - Modules: map[string]compiler.RawModule{ - "entry": { - Manifest: manifest, - // []byte len=0; cap=512 -> default value for empty file - Packages: map[string]compiler.RawPackage{ - "foo": { - "1": make([]byte, 0, 512), - "2": make([]byte, 0, 512), - }, - "foo/bar": { - "3": make([]byte, 0, 512), - }, - "baz": { - "4": make([]byte, 0, 512), - }, - }, - }, - }, - } - - require.Equal(t, expectedBuild, actualBuild) -} - -func createMod(manifest src.Manifest, files map[string][]string, path string) error { - if err := os.MkdirAll(path, 0755); err != nil { //nolint:gofumpt - return err - } - - if err := createFile(path, "neva.yml"); err != nil { - return err - } - - manifestPath := filepath.Join(path, "neva.yml") - - rawManifest, err := yaml.Marshal(manifest) - if err != nil { - return err - } - - if err := os.WriteFile(manifestPath, rawManifest, 0644); err != nil { //nolint:gofumpt - return err - } - - for _, dir := range maps.Keys(files) { - dirPath := filepath.Join(path, dir) - if err := os.MkdirAll(dirPath, 0755); err != nil { //nolint:gofumpt - return err - } - } - - for dir, files := range files { - for _, fileName := range files { - filePath := filepath.Join(path, dir) - if err := createFile(filePath, fileName); err != nil { - return err - } - } - } - - return nil -} - -func createFile(path string, filename string) error { - fullPath := filepath.Join(path, filename) - file, err := os.Create(fullPath) - if err != nil { - return err - } - defer file.Close() - return nil -} diff --git a/internal/compiler/analyzer/analyzer.go b/internal/compiler/analyzer/analyzer.go index c5eb393e..333cc0e7 100644 --- a/internal/compiler/analyzer/analyzer.go +++ b/internal/compiler/analyzer/analyzer.go @@ -5,9 +5,10 @@ import ( "fmt" "strings" + "golang.org/x/exp/maps" + src "github.com/nevalang/neva/internal/compiler/sourcecode" ts "github.com/nevalang/neva/pkg/typesystem" - "golang.org/x/exp/maps" ) type Analyzer struct { @@ -16,7 +17,7 @@ type Analyzer struct { func (a Analyzer) AnalyzeExecutable(mod src.Module, mainPkgName string) (src.Module, error) { if _, ok := mod.Packages[mainPkgName]; !ok { - return src.Module{}, ErrMainPkgNotFound + return src.Module{}, fmt.Errorf("%w: %s", ErrMainPkgNotFound, mainPkgName) } if err := a.mainSpecificPkgValidation(mainPkgName, mod); err != nil { diff --git a/internal/compiler/compiler.go b/internal/compiler/compiler.go index 8bcb7fcc..2d6f441b 100644 --- a/internal/compiler/compiler.go +++ b/internal/compiler/compiler.go @@ -3,6 +3,7 @@ package compiler import ( "context" "fmt" + "strings" src "github.com/nevalang/neva/internal/compiler/sourcecode" "github.com/nevalang/neva/pkg/ir" @@ -40,20 +41,29 @@ type ( } ) -func (c Compiler) Compile(ctx context.Context, build Build, mainPkg string) (*ir.Program, error) { +func (c Compiler) Compile( + ctx context.Context, + build Build, + workdirPath string, + mainPkgName string, +) (*ir.Program, error) { rawMod := build.Modules[build.EntryModule] // TODO support multimodule compilation + if strings.HasPrefix(mainPkgName, "./") { + mainPkgName = strings.TrimPrefix(mainPkgName, "./") + } + parsedPackages, err := c.parser.ParsePackages(ctx, rawMod.Packages) if err != nil { return nil, fmt.Errorf("parse: %w", err) } - + mod := src.Module{ Manifest: rawMod.Manifest, Packages: parsedPackages, } - analyzedProg, err := c.analyzer.AnalyzeExecutable(mod, mainPkg) + analyzedProg, err := c.analyzer.AnalyzeExecutable(mod, mainPkgName) if err != nil { return nil, fmt.Errorf("analyzer: %w", err) } diff --git a/internal/compiler/sourcecode/scope.go b/internal/compiler/sourcecode/scope.go index ee1ff0cc..11eb3f1b 100644 --- a/internal/compiler/sourcecode/scope.go +++ b/internal/compiler/sourcecode/scope.go @@ -10,14 +10,16 @@ import ( // Scope is an entity that can be used to query the program. type Scope struct { - Loc ScopeLocation // It keeps track of current location to properly resolve imports and local references. - Module Module // And of course it does has access to the program itself. + Loc ScopeLocation // It keeps track of current location to properly resolve imports and local references. + // TODO use multiple modules + Module Module // And of course it does has access to the program itself. } // ScopeLocation is used by scope to resolve references. type ScopeLocation struct { - PkgName string // Full (real) name of the current package - FileName string // Name of the current file in the current package + ModuleName string // TODO currently unused + PkgName string // Full (real) name of the current package + FileName string // Name of the current file in the current package } // IsTopType returns true if passed reference is builtin "any" and false otherwise. @@ -97,10 +99,15 @@ var ( ) // Entity returns entity by passed reference. -// If entity is local (ref has no pkg) the current location.pkg is used +// If entity is local (ref has no pkg) the current location.pkg or std/builtin is used // Otherwise we use current file imports to resolve external ref. +// This method MUST be used by all other methods that do entity lookup. func (s Scope) Entity(entityRef EntityRef) (Entity, ScopeLocation, error) { if entityRef.Pkg == "" { + // pkg, ok := s.Module.Packages[s.Loc.PkgName] + // if !ok { + // } + entity, filename, ok := s.Module.Packages[s.Loc.PkgName].Entity(entityRef.Name) if ok { return entity, ScopeLocation{ @@ -109,6 +116,7 @@ func (s Scope) Entity(entityRef EntityRef) (Entity, ScopeLocation, error) { }, nil } + // FIXME std is different module, not just package entity, filename, ok = s.Module.Packages["std/builtin"].Entity(entityRef.Name) if !ok { return Entity{}, ScopeLocation{}, fmt.Errorf("%w: %v", ErrEntityNotFound, entityRef) @@ -125,6 +133,9 @@ func (s Scope) Entity(entityRef EntityRef) (Entity, ScopeLocation, error) { return Entity{}, ScopeLocation{}, fmt.Errorf("%w: %v", ErrNoImport, entityRef.Pkg) } + // TODO update for multi-module workflow + // check that real import package name is in this module, otherwise use the module we need + entity, fileName, err := s.Module.Entity(EntityRef{ Pkg: realImportPkgName, Name: entityRef.Name, diff --git a/internal/compiler/sourcecode/src.go b/internal/compiler/sourcecode/src.go index def54af9..3c5658ef 100644 --- a/internal/compiler/sourcecode/src.go +++ b/internal/compiler/sourcecode/src.go @@ -34,7 +34,7 @@ var ( func (mod Module) Entity(entityRef EntityRef) (entity Entity, filename string, err error) { pkg, ok := mod.Packages[entityRef.Pkg] if !ok { - return Entity{}, "", ErrPkgNotFound + return Entity{}, "", fmt.Errorf("%w: %s", ErrPkgNotFound, entityRef.Pkg) } for filename, file := range pkg { entity, ok := file.Entities[entityRef.Name] diff --git a/internal/interpreter/interpreter.go b/internal/interpreter/interpreter.go index 69eb93ae..7af6babb 100644 --- a/internal/interpreter/interpreter.go +++ b/internal/interpreter/interpreter.go @@ -17,13 +17,13 @@ type Interpreter struct { adapter proto.Adapter } -func (i Interpreter) Interpret(ctx context.Context, pathToMainPkg string) (int, error) { - build, err := i.builder.Build(ctx, pathToMainPkg) +func (i Interpreter) Interpret(ctx context.Context, workdirPath string, mainPkgName string) (int, error) { + build, err := i.builder.Build(ctx, workdirPath) if err != nil { return 0, fmt.Errorf("build: %w", err) } - irProg, err := i.compiler.Compile(ctx, build, pathToMainPkg) + irProg, err := i.compiler.Compile(ctx, build, workdirPath, mainPkgName) if err != nil { return 0, err }