Add more syntax files and include syntax highlighter in the repo

cmd/micro/buffer.go

@@ -16,7 +16,7 @@ import (
 	"unicode/utf8"
 
 	"github.com/mitchellh/go-homedir"
-	"github.com/zyedidia/highlight"
+	"github.com/zyedidia/micro/cmd/micro/highlight"
 )
 
 // Buffer stores the text for files that are loaded into the text editor

cmd/micro/highlight/ftdetect.go

@@ -0,0 +1,19 @@
+package highlight
+
+func DetectFiletype(defs []*Def, filename string, firstLine []byte) *Def {
+	for _, d := range defs {
+		if d.ftdetect[0].Match([]byte(filename)) {
+			return d
+		}
+		if len(d.ftdetect) > 1 {
+			if d.ftdetect[1].Match(firstLine) {
+				return d
+			}
+		}
+	}
+
+	emptyDef := new(Def)
+	emptyDef.FileType = "Unknown"
+	emptyDef.rules = new(Rules)
+	return emptyDef
+}

cmd/micro/highlight/highlighter.go

@@ -0,0 +1,262 @@
+package highlight
+
+import (
+	"regexp"
+	"strings"
+)
+
+func combineLineMatch(src, dst LineMatch) LineMatch {
+	for k, v := range src {
+		if g, ok := dst[k]; ok {
+			if g == "" {
+				dst[k] = v
+			}
+		} else {
+			dst[k] = v
+		}
+	}
+	return dst
+}
+
+type State *Region
+
+type LineStates interface {
+	LineData() [][]byte
+	State(lineN int) State
+	SetState(lineN int, s State)
+	SetMatch(lineN int, m LineMatch)
+}
+
+type Highlighter struct {
+	lastRegion *Region
+	def        *Def
+}
+
+func NewHighlighter(def *Def) *Highlighter {
+	h := new(Highlighter)
+	h.def = def
+	return h
+}
+
+type LineMatch map[int]string
+
+func FindIndex(regex *regexp.Regexp, str []byte, canMatchStart, canMatchEnd bool) []int {
+	regexStr := regex.String()
+	if strings.Contains(regexStr, "^") {
+		if !canMatchStart {
+			return nil
+		}
+	}
+	if strings.Contains(regexStr, "$") {
+		if !canMatchEnd {
+			return nil
+		}
+	}
+	return regex.FindIndex(str)
+}
+
+func FindAllIndex(regex *regexp.Regexp, str []byte, canMatchStart, canMatchEnd bool) [][]int {
+	regexStr := regex.String()
+	if strings.Contains(regexStr, "^") {
+		if !canMatchStart {
+			return nil
+		}
+	}
+	if strings.Contains(regexStr, "$") {
+		if !canMatchEnd {
+			return nil
+		}
+	}
+	return regex.FindAllIndex(str, -1)
+}
+
+func (h *Highlighter) highlightRegion(start int, canMatchEnd bool, lineNum int, line []byte, region *Region) LineMatch {
+	highlights := make(LineMatch)
+
+	if len(line) == 0 {
+		if canMatchEnd {
+			h.lastRegion = region
+		}
+
+		return highlights
+	}
+
+	loc := FindIndex(region.end, line, start == 0, canMatchEnd)
+	if loc != nil {
+		if region.parent == nil {
+			highlights[start+loc[1]] = ""
+			return combineLineMatch(highlights,
+				combineLineMatch(h.highlightRegion(start, false, lineNum, line[:loc[0]], region),
+					h.highlightEmptyRegion(start+loc[1], canMatchEnd, lineNum, line[loc[1]:])))
+		}
+		highlights[start+loc[1]] = region.parent.group
+		return combineLineMatch(highlights,
+			combineLineMatch(h.highlightRegion(start, false, lineNum, line[:loc[0]], region),
+				h.highlightRegion(start+loc[1], canMatchEnd, lineNum, line[loc[1]:], region.parent)))
+	}
+
+	firstLoc := []int{len(line), 0}
+	var firstRegion *Region
+	for _, r := range region.rules.regions {
+		loc := FindIndex(r.start, line, start == 0, canMatchEnd)
+		if loc != nil {
+			if loc[0] < firstLoc[0] {
+				firstLoc = loc
+				firstRegion = r
+			}
+		}
+	}
+	if firstLoc[0] != len(line) {
+		highlights[start+firstLoc[0]] = firstRegion.group
+		return combineLineMatch(highlights,
+			combineLineMatch(h.highlightRegion(start, false, lineNum, line[:firstLoc[0]], region),
+				h.highlightRegion(start+firstLoc[1], canMatchEnd, lineNum, line[firstLoc[1]:], firstRegion)))
+	}
+
+	for _, p := range region.rules.patterns {
+		matches := FindAllIndex(p.regex, line, start == 0, canMatchEnd)
+		for _, m := range matches {
+			highlights[start+m[0]] = p.group
+			if _, ok := highlights[start+m[1]]; !ok {
+				highlights[start+m[1]] = region.group
+			}
+		}
+	}
+
+	if canMatchEnd {
+		h.lastRegion = region
+	}
+
+	return highlights
+}
+
+func (h *Highlighter) highlightEmptyRegion(start int, canMatchEnd bool, lineNum int, line []byte) LineMatch {
+	highlights := make(LineMatch)
+	if len(line) == 0 {
+		if canMatchEnd {
+			h.lastRegion = nil
+		}
+		return highlights
+	}
+
+	firstLoc := []int{len(line), 0}
+	var firstRegion *Region
+	for _, r := range h.def.rules.regions {
+		loc := FindIndex(r.start, line, start == 0, canMatchEnd)
+		if loc != nil {
+			if loc[0] < firstLoc[0] {
+				firstLoc = loc
+				firstRegion = r
+			}
+		}
+	}
+	if firstLoc[0] != len(line) {
+		highlights[start+firstLoc[0]] = firstRegion.group
+		return combineLineMatch(highlights,
+			combineLineMatch(h.highlightEmptyRegion(start, false, lineNum, line[:firstLoc[0]]),
+				h.highlightRegion(start+firstLoc[1], canMatchEnd, lineNum, line[firstLoc[1]:], firstRegion)))
+	}
+
+	for _, p := range h.def.rules.patterns {
+		matches := FindAllIndex(p.regex, line, start == 0, canMatchEnd)
+		for _, m := range matches {
+			highlights[start+m[0]] = p.group
+			if _, ok := highlights[start+m[1]]; !ok {
+				highlights[start+m[1]] = ""
+			}
+		}
+	}
+
+	if canMatchEnd {
+		h.lastRegion = nil
+	}
+
+	return highlights
+}
+
+func (h *Highlighter) HighlightString(input string) []LineMatch {
+	lines := strings.Split(input, "\n")
+	var lineMatches []LineMatch
+
+	for i := 0; i < len(lines); i++ {
+		line := []byte(lines[i])
+
+		if i == 0 || h.lastRegion == nil {
+			lineMatches = append(lineMatches, h.highlightEmptyRegion(0, true, i, line))
+		} else {
+			lineMatches = append(lineMatches, h.highlightRegion(0, true, i, line, h.lastRegion))
+		}
+	}
+
+	return lineMatches
+}
+
+func (h *Highlighter) Highlight(input LineStates, startline int) {
+	lines := input.LineData()
+
+	for i := startline; i < len(lines); i++ {
+		line := []byte(lines[i])
+
+		var match LineMatch
+		if i == 0 || h.lastRegion == nil {
+			match = h.highlightEmptyRegion(0, true, i, line)
+		} else {
+			match = h.highlightRegion(0, true, i, line, h.lastRegion)
+		}
+
+		curState := h.lastRegion
+
+		input.SetMatch(i, match)
+		input.SetState(i, curState)
+	}
+}
+
+func (h *Highlighter) ReHighlightLine(input LineStates, lineN int) {
+	lines := input.LineData()
+
+	line := []byte(lines[lineN])
+
+	h.lastRegion = nil
+	if lineN > 0 {
+		h.lastRegion = input.State(lineN - 1)
+	}
+
+	var match LineMatch
+	if lineN == 0 || h.lastRegion == nil {
+		match = h.highlightEmptyRegion(0, true, lineN, line)
+	} else {
+		match = h.highlightRegion(0, true, lineN, line, h.lastRegion)
+	}
+	curState := h.lastRegion
+
+	input.SetMatch(lineN, match)
+	input.SetState(lineN, curState)
+}
+
+func (h *Highlighter) ReHighlight(input LineStates, startline int) {
+	lines := input.LineData()
+
+	h.lastRegion = nil
+	if startline > 0 {
+		h.lastRegion = input.State(startline - 1)
+	}
+	for i := startline; i < len(lines); i++ {
+		line := []byte(lines[i])
+
+		var match LineMatch
+		if i == 0 || h.lastRegion == nil {
+			match = h.highlightEmptyRegion(0, true, i, line)
+		} else {
+			match = h.highlightRegion(0, true, i, line, h.lastRegion)
+		}
+		curState := h.lastRegion
+		lastState := input.State(i)
+
+		input.SetMatch(i, match)
+		input.SetState(i, curState)
+
+		if curState == lastState {
+			break
+		}
+	}
+}

cmd/micro/highlight/parser.go

@@ -0,0 +1,202 @@
+package highlight
+
+import (
+	"fmt"
+	"regexp"
+
+	"gopkg.in/yaml.v2"
+)
+
+// A Def is a full syntax definition for a language
+// It has a filetype, information about how to detect the filetype based
+// on filename or header (the first line of the file)
+// Then it has the rules which define how to highlight the file
+type Def struct {
+	FileType string
+	ftdetect []*regexp.Regexp
+	rules    *Rules
+}
+
+// A Pattern is one simple syntax rule
+// It has a group that the rule belongs to, as well as
+// the regular expression to match the pattern
+type Pattern struct {
+	group string
+	regex *regexp.Regexp
+}
+
+// Rules defines which patterns and regions can be used to highlight
+// a filetype
+type Rules struct {
+	regions  []*Region
+	patterns []*Pattern
+	includes []string
+}
+
+// A Region is a highlighted region (such as a multiline comment, or a string)
+// It belongs to a group, and has start and end regular expressions
+// A Region also has rules of its own that only apply when matching inside the
+// region and also rules from the above region do not match inside this region
+// Note that a region may contain more regions
+type Region struct {
+	group  string
+	parent *Region
+	start  *regexp.Regexp
+	end    *regexp.Regexp
+	rules  *Rules
+}
+
+// ParseDef parses an input syntax file into a highlight Def
+func ParseDef(input []byte) (s *Def, err error) {
+	// This is just so if we have an error, we can exit cleanly and return the parse error to the user
+	defer func() {
+		if e := recover(); e != nil {
+			err = e.(error)
+		}
+	}()
+
+	var rules map[interface{}]interface{}
+	if err = yaml.Unmarshal(input, &rules); err != nil {
+		return nil, err
+	}
+
+	s = new(Def)
+
+	for k, v := range rules {
+		if k == "filetype" {
+			filetype := v.(string)
+
+			s.FileType = filetype
+		} else if k == "detect" {
+			ftdetect := v.(map[interface{}]interface{})
+			if len(ftdetect) >= 1 {
+				syntax, err := regexp.Compile(ftdetect["filename"].(string))
+				if err != nil {
+					return nil, err
+				}
+
+				s.ftdetect = append(s.ftdetect, syntax)
+			}
+			if len(ftdetect) >= 2 {
+				header, err := regexp.Compile(ftdetect["header"].(string))
+				if err != nil {
+					return nil, err
+				}
+
+				s.ftdetect = append(s.ftdetect, header)
+			}
+		} else if k == "rules" {
+			inputRules := v.([]interface{})
+
+			rules, err := parseRules(inputRules, nil)
+			if err != nil {
+				return nil, err
+			}
+
+			s.rules = rules
+		}
+	}
+
+	return s, err
+}
+
+func ResolveIncludes(defs []*Def) {
+	for _, d := range defs {
+		resolveIncludesInDef(defs, d)
+	}
+}
+
+func resolveIncludesInDef(defs []*Def, d *Def) {
+	for _, lang := range d.rules.includes {
+		for _, searchDef := range defs {
+			if lang == searchDef.FileType {
+				d.rules.patterns = append(d.rules.patterns, searchDef.rules.patterns...)
+				d.rules.regions = append(d.rules.regions, searchDef.rules.regions...)
+			}
+		}
+	}
+	for _, r := range d.rules.regions {
+		resolveIncludesInRegion(defs, r)
+		r.parent = nil
+	}
+}
+
+func resolveIncludesInRegion(defs []*Def, region *Region) {
+	for _, lang := range region.rules.includes {
+		for _, searchDef := range defs {
+			if lang == searchDef.FileType {
+				region.rules.patterns = append(region.rules.patterns, searchDef.rules.patterns...)
+				region.rules.regions = append(region.rules.regions, searchDef.rules.regions...)
+			}
+		}
+	}
+	for _, r := range region.rules.regions {
+		resolveIncludesInRegion(defs, r)
+		r.parent = region
+	}
+}
+
+func parseRules(input []interface{}, curRegion *Region) (*Rules, error) {
+	rules := new(Rules)
+
+	for _, v := range input {
+		rule := v.(map[interface{}]interface{})
+		for k, val := range rule {
+			group := k
+
+			switch object := val.(type) {
+			case string:
+				if k == "include" {
+					rules.includes = append(rules.includes, object)
+				} else {
+					// Pattern
+					r, err := regexp.Compile(object)
+					if err != nil {
+						return nil, err
+					}
+
+					rules.patterns = append(rules.patterns, &Pattern{group.(string), r})
+				}
+			case map[interface{}]interface{}:
+				// Region
+				region, err := parseRegion(group.(string), object, curRegion)
+				if err != nil {
+					return nil, err
+				}
+				rules.regions = append(rules.regions, region)
+			default:
+				return nil, fmt.Errorf("Bad type %T", object)
+			}
+		}
+	}
+
+	return rules, nil
+}
+
+func parseRegion(group string, regionInfo map[interface{}]interface{}, prevRegion *Region) (*Region, error) {
+	var err error
+
+	region := new(Region)
+	region.group = group
+	region.parent = prevRegion
+
+	region.start, err = regexp.Compile(regionInfo["start"].(string))
+
+	if err != nil {
+		return nil, err
+	}
+
+	region.end, err = regexp.Compile(regionInfo["end"].(string))
+
+	if err != nil {
+		return nil, err
+	}
+
+	region.rules, err = parseRules(regionInfo["rules"].([]interface{}), region)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return region, nil
+}

cmd/micro/highlighter.go

@@ -1,6 +1,6 @@
 package main
 
-import "github.com/zyedidia/highlight"
+import "github.com/zyedidia/micro/cmd/micro/highlight"
 
 var syntaxDefs []*highlight.Def
 

cmd/micro/lineArray.go

@@ -5,7 +5,7 @@ import (
 	"io"
 	"unicode/utf8"
 
-	"github.com/zyedidia/highlight"
+	"github.com/zyedidia/micro/cmd/micro/highlight"
 )
 
 func runeToByteIndex(n int, txt []byte) int {