Integration/stage2 3 parser

.gitignore

@@ -101,3 +101,7 @@ fabric.properties
 out
 gen
 *.rdb
+*.db
+cmd/cxbenchmark/bin/cx
+*.out
+*.test

Makefile

@@ -143,3 +143,13 @@ dep: ## Update go vendor
 
 help:
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+benchmark: #Benchmark
+ifndef CXVERSION
+	@echo "$(CXEXE) not found in $(PWD)/bin, please run make install first"
+else
+	mkdir -p $(PWD)/cmd/cxbenchmark/bin/
+	rm -f $(PWD)/cmd/cxbenchmark/bin/$(CXEXE)
+	cp $(PWD)/bin/$(CXEXE) $(PWD)/cmd/cxbenchmark/bin/
+	go test $(PWD)/cmd/cxbenchmark -run BenchmarkCX -tags ptr32 -count=10 -bench=. -benchmem
+endif

cmd/cx/helpers.go

@@ -40,7 +40,7 @@ func parseProgram(options cxCmdFlags, fileNames []string, sourceCode []*os.File)
 	}
 
 	// Parsing all the source code files sent as CLI arguments to CX.
-	cxparsing.ParseSourceCode(sourceCode, fileNames)
+	cxparsing.ParseSourceCode(sourceCode)
 
 	// Checking if a main package exists. If not, create and add it to `AST`.
 	if _, err := actions.AST.GetFunction(constants.MAIN_FUNC, constants.MAIN_PKG); err != nil {

cmd/cxbenchmark/cxbenchmark_test.go

@@ -0,0 +1,18 @@
+package cxbenchmark_test
+
+import (
+	"os/exec"
+	"testing"
+)
+
+func BenchmarkCX(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+
+		cmd := exec.Command("./bin/cx", "./test_files/test.cx")
+		_, err := cmd.CombinedOutput()
+		if err != nil {
+			b.Fatal(err)
+		}
+
+	}
+}

cmd/cxbenchmark/test_files/test.cx

@@ -0,0 +1,79 @@
+package main
+
+var Bool bool = true
+var Byte i8 = 8B
+var I16 i16 = 16H
+var I32 i32 = 32
+var I64 i64 = 64L
+var UByte ui8 = 9UB
+var UI16 ui16 = 17UH
+var UI32 ui32 = 33U
+var UI64 ui64 = 65UL
+var F32 f32 = 0.32
+var F64 f64 = 0.64D
+var string str = "Hello World"
+var Affordance aff
+
+var intArray [5]i32
+// var stringPointer *str
+
+// var abc string	Var in comment
+/*
+	var apple int
+	- Global in a multiline comment
+*/
+
+type CustomType struct {
+	fieldA str
+	fieldB i32
+}
+
+func (customType *CustomType) setFieldA (string str) {
+	customType.fieldA = string
+}
+
+func main () {
+
+	bool.print(Bool)
+	i8.print(Byte)
+	i16.print(I16)
+	i32.print(I32)
+	i64.print(I64)
+	ui8.print(UByte)
+	ui16.print(UI16)
+	ui32.print(UI32)
+	ui64.print(UI64)
+	f32.print(F32)
+	f64.print(F64)
+	str.print(string)	
+
+	//Addition
+	answer := add(I32, 6)
+	i32.print(answer)
+
+	//Multiply
+	var quotient i32
+	var remainder f32
+	quotient, remainder = divide(9, 4)
+	i32.print(quotient)
+	f32.print(remainder)
+
+	printer("Print me")
+}
+
+func add(a i32, b i32)(answer i32) {
+	answer = a + b
+}
+
+func divide(c i32, d i32)(quotient i32, remainder f32) {
+	quotient = c/d
+	remainder = i32.f32(c)%i32.f32(d)
+}
+
+func printer(message str)() {
+	str.print(message)
+}
+
+type AnotherType struct {
+	name str
+}

cmd/cxtest/main.go

@@ -113,7 +113,9 @@ func Execute(c *cli.Context) error {
 	var start = time.Now().Unix()
 
 	fmt.Printf("Running CX tests in dir: '%s'\n", workingDir)
+	timerStart := time.Now()
 	runTestCases(tester)
+	timerTime := time.Since(timerStart)
 	end := time.Now().Unix()
 
 	if runner.Has(logMask, runner.LogTime) {
@@ -124,6 +126,7 @@ func Execute(c *cli.Context) error {
 	fmt.Printf("%d were successful\n", tester.TestSuccess)
 	fmt.Printf("%d failed\n", tester.TestCount-tester.TestSuccess)
 	fmt.Printf("%d skipped\n", tester.TestSkipped)
+	fmt.Printf("Time took %v\n", timerTime)
 
 	if tester.TestCount == 0 || (tester.TestSuccess != tester.TestCount) {
 		return errors.New("not all test succeeded")

cmd/declaration_extractor/declaration_extractor.go

@@ -3,11 +3,36 @@ package declaration_extractor
 import (
 	"fmt"
 	"path/filepath"
+	"regexp"
 	"sync"
-
-	"github.com/skycoin/cx/cmd/packageloader/loader"
 )
 
+//Regexes
+var reName = regexp.MustCompile(`[_a-zA-Z][_a-zA-Z0-9]*`)
+var reEnumInit = regexp.MustCompile(`const\s+\(`)
+var reEnumDec = regexp.MustCompile(`([_a-zA-Z][_a-zA-Z0-9]*)(?:\s+([_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*)){0,1}(?:\s*\=\s*[\s\S]+\S+){0,1}`)
+var reNotSpace = regexp.MustCompile(`\S+`)
+
+// Func Declaration regex for name extraction and syntax checking
+// Components:
+// func - func keyword
+// (?:\s*\(\s*[_a-zA-Z]\w*\s+\*{0,1}\s*[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*\s*\)\s*)|\s+) -  [space/no space] [([reciever object]) [name] | [space]]
+// ([_a-zA-Z]\w*) - name of func
+// (?:\s*\(\s*(?:(?:[_a-zA-Z]\w*\s+\*{0,1}\s*(?:\[(?:[1-9]\d+|[0-9]){0,1}\]\*{0,1}){0,1}\s*[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*\s*,\s*)+[_a-zA-Z]\w*\s+\*{0,1}\s*(?:\[(?:[1-9]\d+|[0-9]){0,1}\]\*{0,1}){0,1}\s*[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*|(?:[_a-zA-Z]\w*\s+\*{0,1}\s*(?:\[(?:[1-9]\d+|[0-9]){0,1}\]\*{0,1}){0,1}\s*[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*){0,1})\s*\)){1,2} - [[space/no space] ([params])]{1,2}
+//		(?:(?:[_a-zA-Z]\w*\s+\*{0,1}\s*(?:\[(?:[1-9]\d+|[0-9]){0,1}\]\*{0,1}){0,1}\s*[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*\s*,\s*)+[_a-zA-Z]\w*\s+\*{0,1}\s*(?:\[(?:[1-9]\d+|[0-9]){0,1}\]\*{0,1}){0,1}\s*[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*|(?:[_a-zA-Z]\w*\s+\*{0,1}\s*(?:\[(?:[1-9]\d+|[0-9]){0,1}\]\*{0,1}){0,1}\s*[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*){0,1}) - [[param name] [data type] [,]]{0,1} [param name] [data type] | [param name] [data type]
+// 			(?:[_a-zA-Z]\w*\s+\*{0,1}\s*(?:\[(?:[1-9]\d+|[0-9]){0,1}\]\*{0,1}){0,1}\s*[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)* - [param name] [*]{0,1} [\[[1-9][0-9]+|[0-9]\][*]{0,1}]{0,1} [word] [[.][word]]*
+//
+// First, finds the func keyword
+// Second, finds out whether the function has a receiver object or not and extracts the func name
+// Third, finds whether there's one or two pairs of parenthesis
+// Forth, finds whether there's one or more params in the parenthesis
+var reFuncDec = regexp.MustCompile(`(func(?:(?:\s*\(\s*[_a-zA-Z]\w*\s+\*{0,1}\s*[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*\s*\)\s*)|\s+)([_a-zA-Z]\w*)(?:\s*\(\s*(?:(?:[_a-zA-Z]\w*(?:\s*\*{0,1}\s*(?:\[(?:[1-9]\d+|[0-9]){0,1}\]\*{0,1}){0,1}\s*|\s+)[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*\s*,\s*)+[_a-zA-Z]\w*(?:\s*\*{0,1}\s*(?:\[(?:[1-9]\d+|[0-9]){0,1}\]\*{0,1}){0,1}|\s+)\s*[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*|(?:[_a-zA-Z]\w*(?:\s*\*{0,1}\s*(?:\[(?:[1-9]\d+|[0-9]){0,1}\]\*{0,1}){0,1}\s*|\s+)[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*){0,1})\s*\)){1,2})(?:\s*{){0,1}`)
+var reGlobalName = regexp.MustCompile(`var\s+([_a-zA-Z]\w*)\s+\*{0,1}\s*(?:\[(?:[1-9]\d+|[0-9]){0,1}\]){0,1}\s*[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*(?:\s*\=\s*[\s\S]+\S+){0,1}`)
+var reStruct = regexp.MustCompile(`type\s+[_a-zA-Z][_a-zA-Z0-9]*\s+struct`)
+var reStructHeader = regexp.MustCompile(`type\s+([_a-zA-Z][_a-zA-Z0-9]*)\s+struct\s*{`)
+var reStructField = regexp.MustCompile(`([_a-zA-Z][_a-zA-Z0-9]*)\s+\*{0,1}\s*(?:\[(?:[1-9]\d+|[0-9]){0,1}\]\*{0,1}){0,1}\s*[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*`)
+var reTypeDefinition = regexp.MustCompile(`type\s+([_a-zA-Z][_a-zA-Z0-9]*)\s+([_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*)`)
+
 func ReDeclarationCheck(Import []ImportDeclaration, Glbl []GlobalDeclaration, Enum []EnumDeclaration, TypeDef []TypeDefinitionDeclaration, Strct []StructDeclaration, Func []FuncDeclaration) error {
 
 	// Checks for the first declaration redeclared
@@ -67,7 +92,17 @@ func ReDeclarationCheck(Import []ImportDeclaration, Glbl []GlobalDeclaration, En
 	for i := 0; i < len(Func); i++ {
 		for j := i + 1; j < len(Func); j++ {
 			if Func[i].FuncName == Func[j].FuncName && Func[i].PackageID == Func[j].PackageID {
-				return fmt.Errorf("%v:%v: redeclaration error: func: %v", filepath.Base(Func[j].FileID), Func[j].LineNumber, Func[i].FuncName)
+				funcMethod1, err := ExtractMethod(Func[i])
+				if err != nil {
+					return err
+				}
+				funcMethod2, err := ExtractMethod(Func[j])
+				if err != nil {
+					return err
+				}
+				if funcMethod1 == funcMethod2 {
+					return fmt.Errorf("%v:%v: redeclaration error: func: %v", filepath.Base(Func[j].FileID), Func[j].LineNumber, Func[i].FuncName)
+				}
 			}
 		}
 	}
@@ -102,7 +137,7 @@ func GetDeclarations(source []byte, Glbls []GlobalDeclaration, Enums []EnumDecla
 	return declarations
 }
 
-func ExtractAllDeclarations(source []*loader.File) ([]ImportDeclaration, []GlobalDeclaration, []EnumDeclaration, []TypeDefinitionDeclaration, []StructDeclaration, []FuncDeclaration, error) {
+func ExtractAllDeclarations(sourceCodeStrings []string, fileNames []string) ([]ImportDeclaration, []GlobalDeclaration, []EnumDeclaration, []TypeDefinitionDeclaration, []StructDeclaration, []FuncDeclaration, error) {
 
 	//Variable declarations
 	var Imports []ImportDeclaration
@@ -113,28 +148,26 @@ func ExtractAllDeclarations(source []*loader.File) ([]ImportDeclaration, []Globa
 	var Funcs []FuncDeclaration
 
 	//Channel declarations
-	importChannel := make(chan []ImportDeclaration, len(source))
-	globalChannel := make(chan []GlobalDeclaration, len(source))
-	enumChannel := make(chan []EnumDeclaration, len(source))
-	typeDefinitionChannel := make(chan []TypeDefinitionDeclaration, len(source))
-	structChannel := make(chan []StructDeclaration, len(source))
-	funcChannel := make(chan []FuncDeclaration, len(source))
-	errorChannel := make(chan error, len(source))
+	importChannel := make(chan []ImportDeclaration, len(sourceCodeStrings))
+	globalChannel := make(chan []GlobalDeclaration, len(sourceCodeStrings))
+	enumChannel := make(chan []EnumDeclaration, len(sourceCodeStrings))
+	typeDefinitionChannel := make(chan []TypeDefinitionDeclaration, len(sourceCodeStrings))
+	structChannel := make(chan []StructDeclaration, len(sourceCodeStrings))
+	funcChannel := make(chan []FuncDeclaration, len(sourceCodeStrings))
+	errorChannel := make(chan error, len(sourceCodeStrings))
 
 	var wg sync.WaitGroup
 
 	// concurrent extractions start
-	for _, currentFile := range source {
+	for i, sourceCode := range sourceCodeStrings {
 
 		wg.Add(1)
 
-		go func(currentFile *loader.File, globalChannel chan<- []GlobalDeclaration, enumChannel chan<- []EnumDeclaration, typeDefinition chan<- []TypeDefinitionDeclaration, structChannel chan<- []StructDeclaration, funcChannel chan<- []FuncDeclaration, errorChannel chan<- error, wg *sync.WaitGroup) {
+		go func(sourceCode string, fileName string, globalChannel chan<- []GlobalDeclaration, enumChannel chan<- []EnumDeclaration, typeDefinition chan<- []TypeDefinitionDeclaration, structChannel chan<- []StructDeclaration, funcChannel chan<- []FuncDeclaration, errorChannel chan<- error, wg *sync.WaitGroup) {
 
 			defer wg.Done()
 
-			srcBytes := currentFile.Content
-
-			fileName := currentFile.FileName
+			srcBytes := []byte(sourceCode)
 			replaceComments := ReplaceCommentsWithWhitespaces(srcBytes)
 			replaceStringContents, err := ReplaceStringContentsWithWhitespaces(replaceComments)
 			if err != nil {
@@ -234,7 +267,7 @@ func ExtractAllDeclarations(source []*loader.File) ([]ImportDeclaration, []Globa
 
 			}(funcChannel, replaceStringContents, fileName, wg)
 
-		}(currentFile, globalChannel, enumChannel, typeDefinitionChannel, structChannel, funcChannel, errorChannel, &wg)
+		}(sourceCode, fileNames[i], globalChannel, enumChannel, typeDefinitionChannel, structChannel, funcChannel, errorChannel, &wg)
 	}
 
 	wg.Wait()