Skip to content

Commit

Permalink
Support jsonobj import command #9
Browse files Browse the repository at this point in the history
  • Loading branch information
lukasmartinelli committed Feb 21, 2016
1 parent 3e5cf29 commit 74c72da
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 10 deletions.
13 changes: 4 additions & 9 deletions import.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,15 +68,10 @@ func (i *Import) Commit() error {
return err
}

err = i.stmt.Close()
if err != nil {
return err
}
// Statement might already be closed
// therefore ignore errors
_ = i.stmt.Close()

err = i.txn.Commit()
if err != nil {
return err
}
return i.txn.Commit()

return nil
}
32 changes: 32 additions & 0 deletions json.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
)

Expand Down Expand Up @@ -66,6 +67,37 @@ func copyJSONRows(i *Import, reader *bufio.Reader, ignoreErrors bool) (error, in
return nil, success, failed
}

func importJSONObject(filename string, connStr string, schema string, tableName string) error {
db, err := connect(connStr, schema)
if err != nil {
return err
}
defer db.Close()

// The entire file is read into memory because we need to add
// it into the PostgreSQL transaction, this will hit memory limits
// for big JSON objects
bytes, err := ioutil.ReadFile(filename)
if err != nil {
return err
}

i, err := NewJSONImport(db, schema, tableName, "data")
if err != nil {
return err
}

// The JSON file is not validated at client side
// it is just copied into the database
// If the JSON file is corrupt PostgreSQL will complain when querying
err = i.AddRow(string(bytes))
if err != nil {
return err
}

return i.Commit()
}

func importJSON(filename string, connStr string, schema string, tableName string, ignoreErrors bool) error {

file, err := os.Open(filename)
Expand Down
22 changes: 21 additions & 1 deletion pgfutter.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func main() {
app.Commands = []cli.Command{
{
Name: "json",
Usage: "Import JSON objects into database",
Usage: "Import newline-delimited JSON objects into database",
Action: func(c *cli.Context) {
cli.CommandHelpTemplate = strings.Replace(cli.CommandHelpTemplate, "[arguments...]", "<json-file>", -1)

Expand All @@ -106,6 +106,26 @@ func main() {
exitOnError(err)
},
},
{
Name: "jsonobj",
Usage: "Import single JSON object into database",
Action: func(c *cli.Context) {
cli.CommandHelpTemplate = strings.Replace(cli.CommandHelpTemplate, "[arguments...]", "<json-file>", -1)

filename := c.Args().First()
if filename == "" {
cli.ShowCommandHelp(c, "jsonobj")
os.Exit(1)
}

schema := c.GlobalString("schema")
tableName := parseTableName(c, filename)

connStr := parseConnStr(c)
err := importJSONObject(filename, connStr, schema, tableName)
exitOnError(err)
},
},
{
Name: "csv",
Usage: "Import CSV into database",
Expand Down

0 comments on commit 74c72da

Please sign in to comment.