Implement installing parsers from github

command/parser.go

@@ -7,8 +7,8 @@ type testCase struct {
 
 type parserConfig struct {
 	Name        string
-	Description string
-	Tests       []testCase
-	Example     string
-	Script      string
+	Description string     `yaml:",omitempty"`
+	Tests       []testCase `yaml:",omitempty"`
+	Example     string     `yaml:",omitempty"`
+	Script      string     `yaml:",flow"`
 }

command/parser_get.go

@@ -19,6 +19,13 @@ func ParserGet(c *cli.Context) error {
 
 	parserName := c.Args().First()
 
+	var outFilePath string
+	if c.IsSet("out") {
+		outFilePath = c.String("out")
+	} else {
+		outFilePath = parserName + ".yaml"
+	}
+
 	src := oauth2.StaticTokenSource(
 		&oauth2.Token{AccessToken: config.Token},
 	)
@@ -29,7 +36,9 @@ func ParserGet(c *cli.Context) error {
 	var q struct {
 		Repository struct {
 			Parser struct {
+				Name       string
 				SourceCode string
+				TestData   []string
 			} `graphql:"parser(name: $parserName)"`
 		} `graphql:"repository(name: $repositoryName)"`
 	}
@@ -42,10 +51,22 @@ func ParserGet(c *cli.Context) error {
 	graphqlErr := client.Query(context.Background(), &q, variables)
 	check(graphqlErr)
 
-	d, yamlErr := yaml.Marshal(&q)
-	check(yamlErr)
+	yamlContent := parserConfig{
+		Name:   q.Repository.Parser.Name,
+		Tests:  Map(q.Repository.Parser.TestData, toTestCase),
+		Script: q.Repository.Parser.SourceCode,
+	}
 
-	ioutil.WriteFile(parserName+".yaml", d, 0644)
+	yamlData, yamlErr := yaml.Marshal(&yamlContent)
+	check(yamlErr)
+	ioutil.WriteFile(outFilePath, yamlData, 0644)
 
 	return nil
 }
+
+func toTestCase(line string) testCase {
+	return testCase{
+		Input:  line,
+		Output: map[string]string{},
+	}
+}

command/parser_add.go → command/parser_push.go

@@ -4,27 +4,41 @@ import (
 	"context"
 	"io/ioutil"
 	"log"
+	"net/http"
 	"strings"
 
 	"github.com/shurcooL/graphql"
 	cli "gopkg.in/urfave/cli.v2"
 	"gopkg.in/yaml.v2"
 )
 
-func ParserAdd(c *cli.Context) error {
+func ParserPush(c *cli.Context) error {
 	config, _ := getServerConfig(c)
 
 	ensureToken(config)
-	ensureRepo(config)
 	ensureURL(config)
 
-	filePath := c.Args().First()
-	file, readErr := ioutil.ReadFile(filePath)
+	var content []byte
+	var readErr error
+
+	repoName := c.Args().Get(0)
+
+	if c.IsSet("file") {
+		filePath := c.String("file")
+		content, readErr = getParserFromFile(filePath)
+	} else if c.IsSet("url") {
+		url := c.String("url")
+		content, readErr = getUrlParser(url)
+	} else {
+		parserName := c.Args().Get(1)
+		content, readErr = getGithubParser(parserName)
+	}
+
 	check(readErr)
 
 	t := parserConfig{}
 
-	err := yaml.Unmarshal(file, &t)
+	err := yaml.Unmarshal(content, &t)
 	check(err)
 
 	client := newGraphQLClient(config)
@@ -38,16 +52,18 @@ func ParserAdd(c *cli.Context) error {
 	tagFields := make([]graphql.String, 0)
 
 	var effectiveName string
-	if c.String("name") != "" {
+	if c.IsSet("name") {
 		effectiveName = c.String("name")
 	} else {
 		effectiveName = t.Name
 	}
 
+	log.Println("NAME:" + effectiveName)
+
 	variables := map[string]interface{}{
 		"name":           graphql.String(effectiveName),
 		"sourceCode":     graphql.String(t.Script),
-		"repositoryName": graphql.String(config.Repo),
+		"repositoryName": graphql.String(repoName),
 		"testData":       testCasesToStrings(t),
 		"tagFields":      tagFields,
 		"force":          graphql.Boolean(true),
@@ -62,6 +78,26 @@ func ParserAdd(c *cli.Context) error {
 	return nil
 }
 
+func getParserFromFile(filePath string) ([]byte, error) {
+	return ioutil.ReadFile(filePath)
+}
+
+func getGithubParser(parserName string) ([]byte, error) {
+	url := "https://raw.githubusercontent.com/humio/community/master/parsers/" + parserName + ".yaml"
+	return getUrlParser(url)
+}
+
+func getUrlParser(url string) ([]byte, error) {
+	response, err := http.Get(url)
+
+	if err != nil {
+		return nil, err
+	}
+
+	defer response.Body.Close()
+	return ioutil.ReadAll(response.Body)
+}
+
 func testCasesToStrings(parser parserConfig) []graphql.String {
 
 	lines := strings.Split(parser.Example, "\n")

command/util.go

@@ -7,3 +7,11 @@ func check(err error) {
 		log.Fatal(err)
 	}
 }
+
+func Map(vs []string, f func(string) testCase) []testCase {
+	vsm := make([]testCase, len(vs))
+	for i, v := range vs {
+		vsm[i] = f(v)
+	}
+	return vsm
+}

main.go

@@ -107,9 +107,16 @@ func main() {
 					{
 						Name:   "get",
 						Action: command.ParserGet,
+						Flags: []cli.Flag{
+							&cli.StringFlag{
+								Name:    "output",
+								Aliases: []string{"out", "o"},
+								Usage:   "The file path where the parser file should be stored.",
+							},
+						},
 					},
 					{
-						Name: "add",
+						Name: "push",
 						Flags: []cli.Flag{
 							&cli.StringFlag{
 								Name:    "name",
@@ -122,7 +129,7 @@ func main() {
 								Usage:   "If a parser exists with the same name update it.",
 							},
 						},
-						Action: command.ParserAdd,
+						Action: command.ParserPush,
 					},
 					{
 						Name:   "remove",