Added Lithuanian language analyzer

analysis/lang/lt/analyzer_lt.go

@@ -0,0 +1,57 @@
+//  Copyright (c) 2019 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lt
+
+import (
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+
+	"github.com/blevesearch/bleve/analysis/token/lowercase"
+	"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
+)
+
+const AnalyzerName = "lt"
+
+func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) {
+	unicodeTokenizer, err := cache.TokenizerNamed(unicode.Name)
+	if err != nil {
+		return nil, err
+	}
+	toLowerFilter, err := cache.TokenFilterNamed(lowercase.Name)
+	if err != nil {
+		return nil, err
+	}
+	stopLtFilter, err := cache.TokenFilterNamed(StopName)
+	if err != nil {
+		return nil, err
+	}
+	stemmerLtFilter, err := cache.TokenFilterNamed(SnowballStemmerName)
+	if err != nil {
+		return nil, err
+	}
+	rv := analysis.Analyzer{
+		Tokenizer: unicodeTokenizer,
+		TokenFilters: []analysis.TokenFilter{
+			toLowerFilter,
+			stopLtFilter,
+			stemmerLtFilter,
+		},
+	}
+	return &rv, nil
+}
+
+func init() {
+	registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)
+}

analysis/lang/lt/analyzer_lt_test.go

@@ -0,0 +1,176 @@
+//  Copyright (c) 2019 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lt
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+)
+
+func TestLithuanianAnalyzer(t *testing.T) {
+	tests := []struct {
+		input  []byte
+		output analysis.TokenStream
+	}{
+		// stemming
+		{
+			input: []byte("kavytė"),
+			output: analysis.TokenStream{
+				&analysis.Token{
+					Term: []byte("kav"),
+				},
+			},
+		},
+		{
+			input: []byte("kavinukas"),
+			output: analysis.TokenStream{
+				&analysis.Token{
+					Term: []byte("kavinuk"),
+				},
+			},
+		},
+		// stop word
+		{
+			input:  []byte("į"),
+			output: analysis.TokenStream{},
+		},
+		// digits safe
+		{
+			input: []byte("Šeši nuliai - 1000000"),
+			output: analysis.TokenStream{
+				&analysis.Token{
+					Term: []byte("šeš"),
+				},
+				&analysis.Token{
+					Term: []byte("nul"),
+				},
+				&analysis.Token{
+					Term: []byte("1000000"),
+				},
+			},
+		},
+		{
+			input: []byte("Tiek savaitgalį, tiek per šventes laukia rudeniški orai: sniego tikėtis neverta"),
+			output: analysis.TokenStream{
+				&analysis.Token{
+					Term: []byte("savaitgal"),
+				},
+				&analysis.Token{
+					Term: []byte("švent"),
+				},
+				&analysis.Token{
+					Term: []byte("lauk"),
+				},
+				&analysis.Token{
+					Term: []byte("rudeniš"),
+				},
+				&analysis.Token{
+					Term: []byte("or"),
+				},
+				&analysis.Token{
+					Term: []byte("snieg"),
+				},
+				&analysis.Token{
+					Term: []byte("tik"),
+				},
+				&analysis.Token{
+					Term: []byte("nevert"),
+				},
+			},
+		},
+		{
+			input: []byte("Visą savaitę prognozuojami klastingi reiškiniai"),
+			output: analysis.TokenStream{
+				&analysis.Token{
+					Term: []byte("vis"),
+				},
+				&analysis.Token{
+					Term: []byte("savait"),
+				},
+				&analysis.Token{
+					// verb. "prognozuo-ti"
+					Term: []byte("prognozuo"),
+				},
+				&analysis.Token{
+					Term: []byte("klast"),
+				},
+				&analysis.Token{
+					Term: []byte("reiškin"),
+				},
+			},
+		},
+		{
+			input: []byte("Susirgęs Arūnas gyvenimui pasirinko šalį, kurioje įteisinta eutanazija: silpsta visos jo organizmo funkcijos"),
+			output: analysis.TokenStream{
+				&analysis.Token{
+					Term: []byte("susirg"),
+				},
+				&analysis.Token{
+					Term: []byte("arūn"),
+				},
+				&analysis.Token{
+					Term: []byte("gyvenim"),
+				},
+				&analysis.Token{
+					Term: []byte("pasirink"),
+				},
+				&analysis.Token{
+					Term: []byte("šal"),
+				},
+				&analysis.Token{
+					Term: []byte("kur"),
+				},
+				&analysis.Token{
+					Term: []byte("įteis"),
+				},
+				&analysis.Token{
+					Term: []byte("eutanazij"),
+				},
+				&analysis.Token{
+					Term: []byte("silpst"),
+				},
+				&analysis.Token{
+					Term: []byte("vis"),
+				},
+				&analysis.Token{
+					Term: []byte("organizm"),
+				},
+				&analysis.Token{
+					Term: []byte("funkcij"),
+				},
+			},
+		},
+	}
+
+	cache := registry.NewCache()
+	analyzer, err := cache.AnalyzerNamed(AnalyzerName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, test := range tests {
+		actual := analyzer.Analyze(test.input)
+		if len(actual) != len(test.output) {
+			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
+		}
+		for i, tok := range actual {
+			if !reflect.DeepEqual(tok.Term, test.output[i].Term) {
+				t.Errorf("expected term %s (% x) got %s (% x)", test.output[i].Term, test.output[i].Term, tok.Term, tok.Term)
+			}
+		}
+	}
+}

analysis/lang/lt/stemmer_lt.go

@@ -0,0 +1,49 @@
+//  Copyright (c) 2019 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lt
+
+import (
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+
+	"github.com/blevesearch/snowballstem"
+	"github.com/blevesearch/snowballstem/lithuanian"
+)
+
+const SnowballStemmerName = "stemmer_lt_snowball"
+
+type LithuanianStemmerFilter struct {
+}
+
+func NewLithuanianStemmerFilter() *LithuanianStemmerFilter {
+	return &LithuanianStemmerFilter{}
+}
+
+func (s *LithuanianStemmerFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
+	for _, token := range input {
+		env := snowballstem.NewEnv(string(token.Term))
+		lithuanian.Stem(env)
+		token.Term = []byte(env.Current())
+	}
+	return input
+}
+
+func LithuanianStemmerFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
+	return NewLithuanianStemmerFilter(), nil
+}
+
+func init() {
+	registry.RegisterTokenFilter(SnowballStemmerName, LithuanianStemmerFilterConstructor)
+}

analysis/lang/lt/stemmer_lt_test.go

@@ -0,0 +1,80 @@
+//  Copyright (c) 2019 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lt
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+)
+
+func TestSnowballLithuanianStemmer(t *testing.T) {
+	tests := []struct {
+		input  analysis.TokenStream
+		output analysis.TokenStream
+	}{
+		{
+			input: analysis.TokenStream{
+				&analysis.Token{
+					Term: []byte("aktorius"),
+				},
+			},
+			output: analysis.TokenStream{
+				&analysis.Token{
+					Term: []byte("aktor"),
+				},
+			},
+		},
+		{
+			input: analysis.TokenStream{
+				&analysis.Token{
+					Term: []byte("kilometrų"),
+				},
+			},
+			output: analysis.TokenStream{
+				&analysis.Token{
+					Term: []byte("kilometr"),
+				},
+			},
+		},
+		{
+			input: analysis.TokenStream{
+				&analysis.Token{
+					Term: []byte("prognozuojami"),
+				},
+			},
+			output: analysis.TokenStream{
+				&analysis.Token{
+					// verb. "prognozuo-ti"
+					Term: []byte("prognozuo"),
+				},
+			},
+		},
+	}
+
+	cache := registry.NewCache()
+	filter, err := cache.TokenFilterNamed(SnowballStemmerName)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, test := range tests {
+		actual := filter.Filter(test.input)
+		if !reflect.DeepEqual(actual, test.output) {
+			t.Errorf("expected %s, got %s", test.output[0].Term, actual[0].Term)
+		}
+	}
+}

analysis/lang/lt/stop_filter_lt.go

@@ -0,0 +1,33 @@
+//  Copyright (c) 2019 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lt
+
+import (
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/analysis/token/stop"
+	"github.com/blevesearch/bleve/registry"
+)
+
+func StopTokenFilterConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.TokenFilter, error) {
+	tokenMap, err := cache.TokenMapNamed(StopName)
+	if err != nil {
+		return nil, err
+	}
+	return stop.NewStopTokensFilter(tokenMap), nil
+}
+
+func init() {
+	registry.RegisterTokenFilter(StopName, StopTokenFilterConstructor)
+}