Support go module

.gitignore

@@ -7,3 +7,5 @@ venv
 lexvec_v1.0.4*
 corpus
 lexvec
+.idea
+output

Makefile

@@ -1,10 +1,10 @@
-CC = clang
+CC = gcc
 CFLAGS = -Ofast -std=gnu99
 OBJ = lexvec
 BUILD = go build --ldflags '-extldflags "-static"' -o $(OBJ)
 
 optimal:
-	CC="$(CC)" CGO_CFLAGS="$(CFLAGS) -march=native" $(BUILD)
+	cd cmd && CC="$(CC)" CGO_CFLAGS="$(CFLAGS) -march=native" $(BUILD) && cd -
 
 cross:
-	CC="$(CC)" CGO_CFLAGS="$(CFLAGS)" $(BUILD)
+	cd cmd && CC="$(CC)" CGO_CFLAGS="$(CFLAGS)" $(BUILD) && cd -

association.go

@@ -20,7 +20,7 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package main
+package lexvec
 
 import "math"
 

cmd/main.go

@@ -0,0 +1,9 @@
+package main
+
+import (
+	"github.com/hjdo/lexvec"
+)
+
+func main() {
+	lexvec.Build()
+}

cooc.go

@@ -20,7 +20,7 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package main
+package lexvec
 
 import (
 	"bufio"

go.mod

@@ -0,0 +1,3 @@
+module github.com/hjdo/lexvec
+
+go 1.14

main.go → lexvec.go

@@ -20,17 +20,19 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package main
+package lexvec
 
 import (
 	"flag"
 	"fmt"
 	"math/rand"
 	"os"
+	"path"
 	"runtime/pprof"
 	"strings"
 )
 
+type OovVectors map[string][]float64
 type (
 	idxUint   = uint32
 	countUint = uint32
@@ -116,7 +118,7 @@ func init() {
 	ctxbreakbytes = []byte(ctxBreakToken)
 }
 
-func main() {
+func Build() {
 	randng = rand.New(rand.NewSource(1))
 	flags := flag.NewFlagSet("default", flag.ExitOnError)
 	flags.StringVar(&corpusPath, "corpus", "", "path to corpus")
@@ -229,3 +231,118 @@ func main() {
 
 	logln(infoLogLevel, "finished!")
 }
+
+func GetOovVectors(words string, subvecsOutputPath string) ([]float64, error) {
+	var (
+		err              error
+		b                = make([]byte, float64Bytes)
+		vector           = make([]float64, 0)
+		matrixBaseOffset int64
+	)
+	subvecsOutput, err := os.Open(subvecsOutputPath)
+	if err != nil {
+		logln(errorLogLevel, "open file failed")
+	}
+	defer subvecsOutput.Close()
+
+	magicNumber := binaryModelReadUint32(subvecsOutput, b)
+	version := binaryModelReadUint32(subvecsOutput, b)
+	vocabSize := binaryModelReadUint32(subvecsOutput, b)
+	subwordMatrixRows := binaryModelReadUint32(subvecsOutput, b)
+	dim := binaryModelReadUint32(subvecsOutput, b)
+	subwordMinN := binaryModelReadUint32(subvecsOutput, b)
+	subwordMaxN := binaryModelReadUint32(subvecsOutput, b)
+
+	if magicNumber != binaryModelMagicNumber {
+		logln(errorLogLevel, "magic number doesnt match")
+	}
+	if version != binaryModelVersion {
+		logln(errorLogLevel, "version number doesnt match")
+	}
+
+	var ivWords []string
+	ivWordToIdx := make(map[string]int)
+	for i := 0; i < int(vocabSize); i++ {
+		wLen := binaryModelReadUint32(subvecsOutput, b)
+
+		b := make([]byte, wLen)
+		if _, err = subvecsOutput.Read(b); err != nil {
+			return vector, nil
+		}
+
+		w := string(b)
+		ivWordToIdx[w] = len(ivWords)
+		ivWords = append(ivWords, w)
+	}
+	if matrixBaseOffset, err = subvecsOutput.Seek(0, 1); err != nil {
+		return vector, nil
+	}
+
+	vec := make([]float64, dim)
+	if len(words) == 0 {
+		return vector, err
+	}
+	parts := strings.Split(words, " ")
+	w := parts[0]
+	var subwords []string
+	if subwordMinN > 0 && len(parts) == 1 {
+		subwords = computeSubwords(w, int(subwordMinN), int(subwordMaxN))
+	} else {
+		subwords = parts[1:]
+	}
+	for j := 0; j < int(dim); j++ {
+		vec[j] = 0
+	}
+	var vLen int
+	if idx, ok := ivWordToIdx[w]; ok {
+		sumVecFromBin(subvecsOutput, matrixBaseOffset, vec, idxUint(idx))
+		vLen++
+	}
+	for _, sw := range subwords {
+		sumVecFromBin(subvecsOutput, matrixBaseOffset, vec, subwordIdx(sw, vocabSize, subwordMatrixRows-vocabSize))
+		vLen++
+	}
+	if vLen > 0 {
+		for j := 0; j < int(dim); j++ {
+			vec[j] /= float64(vLen)
+		}
+	}
+	for _, f := range vec {
+		vector = append(vector, f)
+	}
+	return vector, nil
+}
+
+func StartTrain(outputFolder, corpusP string,
+	dimP idxUint, subsampleP real,
+	minfreqP countUint,
+	modelP, windowP, negativeP, iterationsP, subwordMinNP int) {
+	if _, err := os.Stat(outputFolder); os.IsNotExist(err) {
+		_ = os.Mkdir(outputFolder, 0644)
+	}
+
+	randng = rand.New(rand.NewSource(1))
+	vocabPath = path.Join(outputFolder, "vocab.txt")
+	vectorOutputPath = path.Join(outputFolder, "vectors.txt")
+	subvecsOutputPath = path.Join(outputFolder, "model.bin")
+	corpusPath = corpusP
+	dim = dimP
+	subsample = subsampleP
+	window = windowP
+	negative = negativeP
+	iterations = iterationsP
+	minFreq = minfreqP
+	model = modelP
+	subwordMinN = subwordMinNP
+
+	buildVocab()
+	saveVocab()
+
+	readVocab()
+	processSubwords()
+	buildCoocMatrix()
+	calculateCdsTotalAndLogs()
+	initModel()
+	train(newTrainIteratorIM())
+	saveVectors()
+}

lexvec_test.go

@@ -0,0 +1,80 @@
+package lexvec
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestGetOovVectors(t *testing.T) {
+	type args struct {
+		words         string
+		subvecsOutput string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    OovVectors
+		wantErr bool
+	}{
+		{
+			name: "test",
+			args: args{
+				words:         "test the model",
+				subvecsOutput: "output/model.bin",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := GetOovVectors(tt.args.words, tt.args.subvecsOutput)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("GetOovVectors() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("GetOovVectors() got = %v, want %v", got, tt.want)
+			}
+			t.Log(got)
+		})
+	}
+}
+
+func TestStartTrain(t *testing.T) {
+	type args struct {
+		outputFolder string
+		corpus       string
+		dim          idxUint
+		subsample    real
+		window       int
+		negative     int
+		iterations   int
+		minfreq      countUint
+		model        int
+		subwordMinN  int
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{
+			name: "test",
+			args: args{
+				outputFolder: "output",
+				corpus:       "C:/Users/starsc/Downloads/corpus.txt",
+				dim:          300,
+				subsample:    1e-5,
+				window:       2,
+				negative:     5,
+				iterations:   5,
+				minfreq:      100,
+				model:        0,
+				subwordMinN:  0,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			StartTrain(tt.args.outputFolder, tt.args.corpus, tt.args.dim, tt.args.subsample, tt.args.minfreq, tt.args.model, tt.args.window, tt.args.negative, tt.args.iterations, tt.args.subwordMinN)
+		})
+	}
+}

matrix.go

@@ -20,7 +20,7 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package main
+package lexvec
 
 type matrix interface {
 	get(row, col idxUint) countUint

model.go

@@ -20,7 +20,7 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package main
+package lexvec
 
 import (
 	"bufio"

sampling.go

@@ -20,7 +20,7 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package main
+package lexvec
 
 import (
 	"math"

sgd.go

@@ -20,7 +20,7 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package main
+package lexvec
 
 // #include "sgdcgo.h"
 import "C"

text.go

@@ -20,7 +20,7 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package main
+package lexvec
 
 import (
 	"bufio"

train.go

@@ -20,7 +20,7 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package main
+package lexvec
 
 import (
 	"bufio"

utils.go

@@ -20,7 +20,7 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package main
+package lexvec
 
 import (
 	"errors"
@@ -31,7 +31,7 @@ import (
 // Helper for aborting on error.
 func check(e error) {
 	if e != nil {
-		panic(e)
+		logln(errorLogLevel, "panic: %v", e)
 	}
 }
 
@@ -62,7 +62,7 @@ func doLog(level int, msg string, lineBreak bool, args ...interface{}) {
 	}
 	os.Stderr.Sync()
 	if level == errorLogLevel {
-		os.Exit(1)
+		//os.Exit(1)
 	}
 }
 

vocab.go

@@ -20,7 +20,7 @@
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-package main
+package lexvec
 
 import (
 	"bufio"