Switch to vfsgen for runtime generation

internal/config/rtfiles.go

@@ -1,3 +1,4 @@
+//go:generate go run runtime_generate.go ../../runtime/syntax ../../runtime
 package config
 
 import (
@@ -9,6 +10,8 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+
+	"github.com/zyedidia/micro/v2/internal/vfsutil"
 )
 
 const (
@@ -90,7 +93,7 @@ func (af assetFile) Name() string {
 }
 
 func (af assetFile) Data() ([]byte, error) {
-	return Asset(string(af))
+	return vfsutil.ReadFile(Assets, string(af))
 }
 
 func (nf namedFile) Name() string {
@@ -111,7 +114,10 @@ func AddRealRuntimeFile(fileType RTFiletype, file RuntimeFile) {
 // AddRuntimeFilesFromDirectory registers each file from the given directory for
 // the filetype which matches the file-pattern
 func AddRuntimeFilesFromDirectory(fileType RTFiletype, directory, pattern string) {
-	files, _ := ioutil.ReadDir(directory)
+	files, err := ioutil.ReadDir(directory)
+	if err != nil {
+		return
+	}
 	for _, f := range files {
 		if ok, _ := filepath.Match(pattern, f.Name()); !f.IsDir() && ok {
 			fullPath := filepath.Join(directory, f.Name())
@@ -123,13 +129,13 @@ func AddRuntimeFilesFromDirectory(fileType RTFiletype, directory, pattern string
 // AddRuntimeFilesFromAssets registers each file from the given asset-directory for
 // the filetype which matches the file-pattern
 func AddRuntimeFilesFromAssets(fileType RTFiletype, directory, pattern string) {
-	files, err := AssetDir(directory)
+	files, err := vfsutil.ReadDir(Assets, directory)
 	if err != nil {
 		return
 	}
 	for _, f := range files {
-		if ok, _ := path.Match(pattern, f); ok {
-			AddRuntimeFile(fileType, assetFile(path.Join(directory, f)))
+		if ok, _ := path.Match(pattern, f.Name()); ok {
+			AddRuntimeFile(fileType, assetFile(path.Join(directory, f.Name())))
 		}
 	}
 }
@@ -156,11 +162,60 @@ func ListRealRuntimeFiles(fileType RTFiletype) []RuntimeFile {
 	return realFiles[fileType]
 }
 
+func addPlugin(dirname string, plugdir string, isID func(string) bool, vfs bool) {
+	var srcs []os.FileInfo
+	var err error
+	if vfs {
+		srcs, err = vfsutil.ReadDir(Assets, filepath.Join(plugdir, dirname))
+	} else {
+		srcs, err = ioutil.ReadDir(filepath.Join(plugdir, dirname))
+	}
+	if err != nil {
+		return
+	}
+	p := new(Plugin)
+	p.Name = dirname
+	p.DirName = dirname
+	p.Default = vfs
+	for _, f := range srcs {
+		if strings.HasSuffix(f.Name(), ".lua") {
+			if vfs {
+				p.Srcs = append(p.Srcs, assetFile(filepath.Join(plugdir, dirname, f.Name())))
+			} else {
+				p.Srcs = append(p.Srcs, realFile(filepath.Join(plugdir, dirname, f.Name())))
+			}
+		} else if strings.HasSuffix(f.Name(), ".json") {
+			var data []byte
+			var err error
+
+			if vfs {
+				data, err = vfsutil.ReadFile(Assets, filepath.Join(plugdir, dirname, f.Name()))
+			} else {
+				data, err = ioutil.ReadFile(filepath.Join(plugdir, dirname, f.Name()))
+			}
+			if err != nil {
+				continue
+			}
+			p.Info, err = NewPluginInfo(data)
+			if err != nil {
+				continue
+			}
+			p.Name = p.Info.Name
+		}
+	}
+
+	if !isID(p.Name) || len(p.Srcs) <= 0 {
+		log.Println(p.Name, "is not a plugin")
+		return
+	}
+	Plugins = append(Plugins, p)
+}
+
 // InitRuntimeFiles initializes all assets file and the config directory
 func InitRuntimeFiles() {
 	add := func(fileType RTFiletype, dir, pattern string) {
 		AddRuntimeFilesFromDirectory(fileType, filepath.Join(ConfigDir, dir), pattern)
-		AddRuntimeFilesFromAssets(fileType, path.Join("runtime", dir), pattern)
+		AddRuntimeFilesFromAssets(fileType, dir, pattern)
 	}
 
 	add(RTColorscheme, "colorschemes", "*.micro")
@@ -179,68 +234,24 @@ func InitRuntimeFiles() {
 
 	// Search ConfigDir for plugin-scripts
 	plugdir := filepath.Join(ConfigDir, "plug")
-	files, _ := ioutil.ReadDir(plugdir)
+	files, err := ioutil.ReadDir(plugdir)
 
 	isID := regexp.MustCompile(`^[_A-Za-z0-9]+$`).MatchString
 
-	for _, d := range files {
-		if d.IsDir() {
-			srcs, _ := ioutil.ReadDir(filepath.Join(plugdir, d.Name()))
-			p := new(Plugin)
-			p.Name = d.Name()
-			p.DirName = d.Name()
-			for _, f := range srcs {
-				if strings.HasSuffix(f.Name(), ".lua") {
-					p.Srcs = append(p.Srcs, realFile(filepath.Join(plugdir, d.Name(), f.Name())))
-				} else if strings.HasSuffix(f.Name(), ".json") {
-					data, err := ioutil.ReadFile(filepath.Join(plugdir, d.Name(), f.Name()))
-					if err != nil {
-						continue
-					}
-					p.Info, err = NewPluginInfo(data)
-					if err != nil {
-						continue
-					}
-					p.Name = p.Info.Name
-				}
+	if err == nil {
+		for _, d := range files {
+			if d.IsDir() {
+				addPlugin(d.Name(), plugdir, isID, false)
 			}
-
-			if !isID(p.Name) || len(p.Srcs) <= 0 {
-				log.Println(p.Name, "is not a plugin")
-				continue
-			}
-			Plugins = append(Plugins, p)
 		}
 	}
 
-	plugdir = filepath.Join("runtime", "plugins")
-	if files, err := AssetDir(plugdir); err == nil {
+	plugdir = "plugins"
+	files, err = vfsutil.ReadDir(Assets, plugdir)
+	if err == nil {
 		for _, d := range files {
-			if srcs, err := AssetDir(filepath.Join(plugdir, d)); err == nil {
-				p := new(Plugin)
-				p.Name = d
-				p.DirName = d
-				p.Default = true
-				for _, f := range srcs {
-					if strings.HasSuffix(f, ".lua") {
-						p.Srcs = append(p.Srcs, assetFile(filepath.Join(plugdir, d, f)))
-					} else if strings.HasSuffix(f, ".json") {
-						data, err := Asset(filepath.Join(plugdir, d, f))
-						if err != nil {
-							continue
-						}
-						p.Info, err = NewPluginInfo(data)
-						if err != nil {
-							continue
-						}
-						p.Name = p.Info.Name
-					}
-				}
-				if !isID(p.Name) || len(p.Srcs) <= 0 {
-					log.Println(p.Name, "is not a plugin")
-					continue
-				}
-				Plugins = append(Plugins, p)
+			if d.IsDir() {
+				addPlugin(d.Name(), plugdir, isID, true)
 			}
 		}
 	}
@@ -277,7 +288,7 @@ func PluginAddRuntimeFile(plugin string, filetype RTFiletype, filePath string) e
 	if _, err := os.Stat(fullpath); err == nil {
 		AddRealRuntimeFile(filetype, realFile(fullpath))
 	} else {
-		fullpath = path.Join("runtime", "plugins", pldir, filePath)
+		fullpath = path.Join("plugins", pldir, filePath)
 		AddRuntimeFile(filetype, assetFile(fullpath))
 	}
 	return nil
@@ -294,7 +305,7 @@ func PluginAddRuntimeFilesFromDirectory(plugin string, filetype RTFiletype, dire
 	if _, err := os.Stat(fullpath); err == nil {
 		AddRuntimeFilesFromDirectory(filetype, fullpath, pattern)
 	} else {
-		fullpath = path.Join("runtime", "plugins", pldir, directory)
+		fullpath = path.Join("plugins", pldir, directory)
 		AddRuntimeFilesFromAssets(filetype, fullpath, pattern)
 	}
 	return nil

internal/config/runtime_generate.go

@@ -0,0 +1,112 @@
+// +build ignore
+
+// This script generates the embedded runtime filesystem, and also creates
+// syntax header metadata which makes loading syntax files at runtime faster
+// Invoke as go run runtime_generate.go syntaxDir runtimeDir
+
+package main
+
+import (
+	"bytes"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"strings"
+
+	yaml "gopkg.in/yaml.v2"
+
+	"github.com/shurcooL/vfsgen"
+)
+
+type HeaderYaml struct {
+	FileType string `yaml:"filetype"`
+	Detect   struct {
+		FNameRgx  string `yaml:"filename"`
+		HeaderRgx string `yaml:"header"`
+	} `yaml:"detect"`
+}
+
+type Header struct {
+	FileType  string
+	FNameRgx  string
+	HeaderRgx string
+}
+
+func convert(name string) {
+	filename := name + ".yaml"
+	var hdr HeaderYaml
+	source, err := ioutil.ReadFile(filename)
+	if err != nil {
+		panic(err)
+	}
+	err = yaml.Unmarshal(source, &hdr)
+	if err != nil {
+		panic(err)
+	}
+	encode(name, hdr)
+}
+
+func encode(name string, c HeaderYaml) {
+	f, _ := os.Create(name + ".hdr")
+	f.WriteString(c.FileType + "\n")
+	f.WriteString(c.Detect.FNameRgx + "\n")
+	f.WriteString(c.Detect.HeaderRgx + "\n")
+	f.Close()
+}
+
+func decode(name string) Header {
+	data, _ := ioutil.ReadFile(name + ".hdr")
+	strs := bytes.Split(data, []byte{'\n'})
+	var hdr Header
+	hdr.FileType = string(strs[0])
+	hdr.FNameRgx = string(strs[1])
+	hdr.HeaderRgx = string(strs[2])
+
+	return hdr
+}
+
+func main() {
+	orig, err := os.Getwd()
+	if err != nil {
+		log.Fatalln("Couldn't get cwd")
+		return
+	}
+	if len(os.Args) < 2 {
+		log.Fatalln("Not enough arguments")
+	}
+
+	syntaxDir := os.Args[1]
+	assetDir := os.Args[2]
+
+	os.Chdir(syntaxDir)
+	files, _ := ioutil.ReadDir(".")
+
+	// first remove all existing header files (clean the directory)
+	for _, f := range files {
+		fname := f.Name()
+		if strings.HasSuffix(fname, ".hdr") {
+			os.Remove(fname)
+		}
+	}
+
+	// now create a header file for each yaml
+	for _, f := range files {
+		fname := f.Name()
+		if strings.HasSuffix(fname, ".yaml") {
+			convert(fname[:len(fname)-5])
+		}
+	}
+
+	// create the assets_vfsdata.go file for embedding in the binary
+	os.Chdir(orig)
+
+	var assets http.FileSystem = http.Dir(assetDir)
+	err = vfsgen.Generate(assets, vfsgen.Options{
+		PackageName:  "config",
+		VariableName: "Assets",
+	})
+	if err != nil {
+		log.Fatalln(err)
+	}
+}

internal/vfsutil/file.go

@@ -0,0 +1,21 @@
+package vfsutil
+
+import (
+	"net/http"
+	"os"
+)
+
+// File implements http.FileSystem using the native file system restricted to a
+// specific file served at root.
+//
+// While the FileSystem.Open method takes '/'-separated paths, a File's string
+// value is a filename on the native file system, not a URL, so it is separated
+// by filepath.Separator, which isn't necessarily '/'.
+type File string
+
+func (f File) Open(name string) (http.File, error) {
+	if name != "/" {
+		return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrNotExist}
+	}
+	return os.Open(string(f))
+}

internal/vfsutil/vfsutil.go

@@ -0,0 +1,39 @@
+// Package vfsutil implements some I/O utility functions for http.FileSystem.
+package vfsutil
+
+import (
+	"io/ioutil"
+	"net/http"
+	"os"
+)
+
+// ReadDir reads the contents of the directory associated with file and
+// returns a slice of FileInfo values in directory order.
+func ReadDir(fs http.FileSystem, name string) ([]os.FileInfo, error) {
+	f, err := fs.Open(name)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return f.Readdir(0)
+}
+
+// Stat returns the FileInfo structure describing file.
+func Stat(fs http.FileSystem, name string) (os.FileInfo, error) {
+	f, err := fs.Open(name)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return f.Stat()
+}
+
+// ReadFile reads the file named by path from fs and returns the contents.
+func ReadFile(fs http.FileSystem, path string) ([]byte, error) {
+	rc, err := fs.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer rc.Close()
+	return ioutil.ReadAll(rc)
+}

internal/vfsutil/walk.go

@@ -0,0 +1,146 @@
+package vfsutil
+
+import (
+	"io"
+	"net/http"
+	"os"
+	pathpkg "path"
+	"path/filepath"
+	"sort"
+)
+
+// Walk walks the filesystem rooted at root, calling walkFn for each file or
+// directory in the filesystem, including root. All errors that arise visiting files
+// and directories are filtered by walkFn. The files are walked in lexical
+// order.
+func Walk(fs http.FileSystem, root string, walkFn filepath.WalkFunc) error {
+	info, err := Stat(fs, root)
+	if err != nil {
+		return walkFn(root, nil, err)
+	}
+	return walk(fs, root, info, walkFn)
+}
+
+// readDirNames reads the directory named by dirname and returns
+// a sorted list of directory entries.
+func readDirNames(fs http.FileSystem, dirname string) ([]string, error) {
+	fis, err := ReadDir(fs, dirname)
+	if err != nil {
+		return nil, err
+	}
+	names := make([]string, len(fis))
+	for i := range fis {
+		names[i] = fis[i].Name()
+	}
+	sort.Strings(names)
+	return names, nil
+}
+
+// walk recursively descends path, calling walkFn.
+func walk(fs http.FileSystem, path string, info os.FileInfo, walkFn filepath.WalkFunc) error {
+	err := walkFn(path, info, nil)
+	if err != nil {
+		if info.IsDir() && err == filepath.SkipDir {
+			return nil
+		}
+		return err
+	}
+
+	if !info.IsDir() {
+		return nil
+	}
+
+	names, err := readDirNames(fs, path)
+	if err != nil {
+		return walkFn(path, info, err)
+	}
+
+	for _, name := range names {
+		filename := pathpkg.Join(path, name)
+		fileInfo, err := Stat(fs, filename)
+		if err != nil {
+			if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir {
+				return err
+			}
+		} else {
+			err = walk(fs, filename, fileInfo, walkFn)
+			if err != nil {
+				if !fileInfo.IsDir() || err != filepath.SkipDir {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// WalkFilesFunc is the type of the function called for each file or directory visited by WalkFiles.
+// It's like filepath.WalkFunc, except it provides an additional ReadSeeker parameter for file being visited.
+type WalkFilesFunc func(path string, info os.FileInfo, rs io.ReadSeeker, err error) error
+
+// WalkFiles walks the filesystem rooted at root, calling walkFn for each file or
+// directory in the filesystem, including root. In addition to FileInfo, it passes an
+// ReadSeeker to walkFn for each file it visits.
+func WalkFiles(fs http.FileSystem, root string, walkFn WalkFilesFunc) error {
+	file, info, err := openStat(fs, root)
+	if err != nil {
+		return walkFn(root, nil, nil, err)
+	}
+	return walkFiles(fs, root, info, file, walkFn)
+}
+
+// walkFiles recursively descends path, calling walkFn.
+// It closes the input file after it's done with it, so the caller shouldn't.
+func walkFiles(fs http.FileSystem, path string, info os.FileInfo, file http.File, walkFn WalkFilesFunc) error {
+	err := walkFn(path, info, file, nil)
+	file.Close()
+	if err != nil {
+		if info.IsDir() && err == filepath.SkipDir {
+			return nil
+		}
+		return err
+	}
+
+	if !info.IsDir() {
+		return nil
+	}
+
+	names, err := readDirNames(fs, path)
+	if err != nil {
+		return walkFn(path, info, nil, err)
+	}
+
+	for _, name := range names {
+		filename := pathpkg.Join(path, name)
+		file, fileInfo, err := openStat(fs, filename)
+		if err != nil {
+			if err := walkFn(filename, nil, nil, err); err != nil && err != filepath.SkipDir {
+				return err
+			}
+		} else {
+			err = walkFiles(fs, filename, fileInfo, file, walkFn)
+			// file is closed by walkFiles, so we don't need to close it here.
+			if err != nil {
+				if !fileInfo.IsDir() || err != filepath.SkipDir {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// openStat performs Open and Stat and returns results, or first error encountered.
+// The caller is responsible for closing the returned file when done.
+func openStat(fs http.FileSystem, name string) (http.File, os.FileInfo, error) {
+	f, err := fs.Open(name)
+	if err != nil {
+		return nil, nil, err
+	}
+	fi, err := f.Stat()
+	if err != nil {
+		f.Close()
+		return nil, nil, err
+	}
+	return f, fi, nil
+}

internal/vfsutil/walk_test.go

@@ -0,0 +1,93 @@
+package vfsutil_test
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+
+	"github.com/shurcooL/httpfs/vfsutil"
+	"golang.org/x/tools/godoc/vfs/httpfs"
+	"golang.org/x/tools/godoc/vfs/mapfs"
+)
+
+func ExampleWalk() {
+	var fs http.FileSystem = httpfs.New(mapfs.New(map[string]string{
+		"zzz-last-file.txt":   "It should be visited last.",
+		"a-file.txt":          "It has stuff.",
+		"another-file.txt":    "Also stuff.",
+		"folderA/entry-A.txt": "Alpha.",
+		"folderA/entry-B.txt": "Beta.",
+	}))
+
+	walkFn := func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			log.Printf("can't stat file %s: %v\n", path, err)
+			return nil
+		}
+		fmt.Println(path)
+		return nil
+	}
+
+	err := vfsutil.Walk(fs, "/", walkFn)
+	if err != nil {
+		panic(err)
+	}
+
+	// Output:
+	// /
+	// /a-file.txt
+	// /another-file.txt
+	// /folderA
+	// /folderA/entry-A.txt
+	// /folderA/entry-B.txt
+	// /zzz-last-file.txt
+}
+
+func ExampleWalkFiles() {
+	var fs http.FileSystem = httpfs.New(mapfs.New(map[string]string{
+		"zzz-last-file.txt":   "It should be visited last.",
+		"a-file.txt":          "It has stuff.",
+		"another-file.txt":    "Also stuff.",
+		"folderA/entry-A.txt": "Alpha.",
+		"folderA/entry-B.txt": "Beta.",
+	}))
+
+	walkFn := func(path string, fi os.FileInfo, r io.ReadSeeker, err error) error {
+		if err != nil {
+			log.Printf("can't stat file %s: %v\n", path, err)
+			return nil
+		}
+		fmt.Println(path)
+		if !fi.IsDir() {
+			b, err := ioutil.ReadAll(r)
+			if err != nil {
+				log.Printf("can't read file %s: %v\n", path, err)
+				return nil
+			}
+			fmt.Printf("%q\n", b)
+		}
+		return nil
+	}
+
+	err := vfsutil.WalkFiles(fs, "/", walkFn)
+	if err != nil {
+		panic(err)
+	}
+
+	// Output:
+	// /
+	// /a-file.txt
+	// "It has stuff."
+	// /another-file.txt
+	// "Also stuff."
+	// /folderA
+	// /folderA/entry-A.txt
+	// "Alpha."
+	// /folderA/entry-B.txt
+	// "Beta."
+	// /zzz-last-file.txt
+	// "It should be visited last."
+}