Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve type-inference performance #809

Merged
merged 16 commits into from
Dec 27, 2021
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
extend type-inference unit-test cases
johnkerl committed Dec 26, 2021
commit 7c753886b978d4135eb74468bb978b7d63338692
1 change: 1 addition & 0 deletions .vimrc
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
map \d :w<C-m>:!clear;echo Building ...; echo; make mlr<C-m>
map \f :w<C-m>:!clear;echo Building ...; echo; make ut<C-m>
map \r :w<C-m>:!clear;echo Building ...; echo; make mv-ut<C-m>
9 changes: 9 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
@@ -31,6 +31,15 @@ install: build
unit-test ut:
go test github.com/johnkerl/miller/internal/pkg/...

lib-ut:
go test github.com/johnkerl/miller/internal/pkg/lib...
mv-ut:
go test github.com/johnkerl/miller/internal/pkg/mlrval/...
bifs-ut:
go test github.com/johnkerl/miller/internal/pkg/bifs/...
input-ut:
go test github.com/johnkerl/miller/internal/pkg/input/...

# ----------------------------------------------------------------
# Regression tests (large number)
#
259 changes: 259 additions & 0 deletions internal/pkg/mlrval/mlrval_infer_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,259 @@
// ================================================================
// Tests mlrval constructors.
// ================================================================

package mlrval

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestInferWithOctalAsString(t *testing.T) {
assert.True(t, inferWithOctalAsString(FromDeferredType(""), false).IsVoid())

assert.True(t, inferWithOctalAsString(FromDeferredType("true"), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("false"), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("true"), true).IsBool())
assert.True(t, inferWithOctalAsString(FromDeferredType("false"), true).IsBool())

assert.True(t, inferWithOctalAsString(FromDeferredType("abc"), false).IsString())

assert.True(t, inferWithOctalAsString(FromDeferredType("0123"), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("-0123"), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("0377"), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("-0377"), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("0923"), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("-0923"), false).IsString())

assert.True(t, inferWithOctalAsString(FromDeferredType("123"), false).IsInt())
assert.True(t, inferWithOctalAsString(FromDeferredType("-123"), false).IsInt())
assert.True(t, inferWithOctalAsString(FromDeferredType("0xff"), false).IsInt())
assert.True(t, inferWithOctalAsString(FromDeferredType("-0xff"), false).IsInt())
assert.True(t, inferWithOctalAsString(FromDeferredType("0b1011"), false).IsInt())
assert.True(t, inferWithOctalAsString(FromDeferredType("-0b1011"), false).IsInt())
assert.True(t, inferWithOctalAsString(FromDeferredType("0x7fffffffffffffff"), false).IsInt())
assert.True(t, inferWithOctalAsString(FromDeferredType("0x8000000000000000"), false).IsInt())
assert.True(t, inferWithOctalAsString(FromDeferredType("0xffffffffffffffff"), false).IsInt())

assert.True(t, inferWithOctalAsString(FromDeferredType("12_3"), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("-12_3"), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("1_2.3_4"), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("-1_2.3_4"), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("0xca_fe"), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("-0xca_fe"), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("0b1011_1101"), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("-0b1011_1101"), false).IsString())

assert.True(t, inferWithOctalAsString(FromDeferredType("."), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("-."), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("123."), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType("-123."), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType(".123"), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType("-.123"), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType("123.456"), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType("-123.456"), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType("1e2."), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("-1e2."), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("1e-2."), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("-1e-2."), false).IsString())
assert.True(t, inferWithOctalAsString(FromDeferredType("1.2e3"), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType("-1.2e3"), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType("1.2e-3"), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType("-1.2e-3"), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType("1.e3"), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType("-1.e3"), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType("1.e-3"), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType("-1.e-3"), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType(".2e3"), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType("-.2e3"), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType(".2e-3"), false).IsFloat())
assert.True(t, inferWithOctalAsString(FromDeferredType("-.2e-3"), false).IsFloat())
}

func TestInferWithOctalAsInt(t *testing.T) {
assert.True(t, inferWithOctalAsInt(FromDeferredType(""), false).IsVoid())

assert.True(t, inferWithOctalAsInt(FromDeferredType("true"), false).IsString())
assert.True(t, inferWithOctalAsInt(FromDeferredType("false"), false).IsString())
assert.True(t, inferWithOctalAsInt(FromDeferredType("true"), true).IsBool())
assert.True(t, inferWithOctalAsInt(FromDeferredType("false"), true).IsBool())

assert.True(t, inferWithOctalAsInt(FromDeferredType("abc"), false).IsString())

assert.True(t, inferWithOctalAsInt(FromDeferredType("0123"), false).IsInt())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-0123"), false).IsInt())
assert.True(t, inferWithOctalAsInt(FromDeferredType("0377"), false).IsInt())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-0377"), false).IsInt())
assert.True(t, inferWithOctalAsInt(FromDeferredType("0923"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-0923"), false).IsFloat())

assert.True(t, inferWithOctalAsInt(FromDeferredType("123"), false).IsInt())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-123"), false).IsInt())
assert.True(t, inferWithOctalAsInt(FromDeferredType("0xff"), false).IsInt())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-0xff"), false).IsInt())
assert.True(t, inferWithOctalAsInt(FromDeferredType("0b1011"), false).IsInt())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-0b1011"), false).IsInt())
assert.True(t, inferWithOctalAsInt(FromDeferredType("0x7fffffffffffffff"), false).IsInt())
assert.True(t, inferWithOctalAsInt(FromDeferredType("0x8000000000000000"), false).IsInt())
assert.True(t, inferWithOctalAsInt(FromDeferredType("0xffffffffffffffff"), false).IsInt())

assert.True(t, inferWithOctalAsInt(FromDeferredType("12_3"), false).IsString())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-12_3"), false).IsString())
assert.True(t, inferWithOctalAsInt(FromDeferredType("1_2.3_4"), false).IsString())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-1_2.3_4"), false).IsString())
assert.True(t, inferWithOctalAsInt(FromDeferredType("0xca_fe"), false).IsString())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-0xca_fe"), false).IsString())
assert.True(t, inferWithOctalAsInt(FromDeferredType("0b1011_1101"), false).IsString())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-0b1011_1101"), false).IsString())

assert.True(t, inferWithOctalAsInt(FromDeferredType("."), false).IsString())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-."), false).IsString())
assert.True(t, inferWithOctalAsInt(FromDeferredType("123."), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-123."), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType(".123"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-.123"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("123.456"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-123.456"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("1e2."), false).IsString())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-1e2."), false).IsString())
assert.True(t, inferWithOctalAsInt(FromDeferredType("1e-2."), false).IsString())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-1e-2."), false).IsString())
assert.True(t, inferWithOctalAsInt(FromDeferredType("1.2e3"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-1.2e3"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("1.2e-3"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-1.2e-3"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("1.e3"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-1.e3"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("1.e-3"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-1.e-3"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType(".2e3"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-.2e3"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType(".2e-3"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-.2e-3"), false).IsFloat())
}

func TestInferWithIntAsFloat(t *testing.T) {
assert.True(t, inferWithIntAsFloat(FromDeferredType(""), false).IsVoid())

assert.True(t, inferWithIntAsFloat(FromDeferredType("true"), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("false"), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("true"), true).IsBool())
assert.True(t, inferWithIntAsFloat(FromDeferredType("false"), true).IsBool())

assert.True(t, inferWithIntAsFloat(FromDeferredType("abc"), false).IsString())

assert.True(t, inferWithIntAsFloat(FromDeferredType("0123"), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-0123"), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("0377"), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-0377"), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("0923"), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-0923"), false).IsString())

assert.True(t, inferWithIntAsFloat(FromDeferredType("123"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-123"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("0xff"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-0xff"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("0b1011"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-0b1011"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("0x7fffffffffffffff"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("0x8000000000000000"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("0xffffffffffffffff"), false).IsFloat())

assert.True(t, inferWithIntAsFloat(FromDeferredType("12_3"), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-12_3"), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("1_2.3_4"), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-1_2.3_4"), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("0xca_fe"), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-0xca_fe"), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("0b1011_1101"), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-0b1011_1101"), false).IsString())

assert.True(t, inferWithIntAsFloat(FromDeferredType("."), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-."), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("123."), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-123."), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType(".123"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-.123"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("123.456"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-123.456"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("1e2."), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-1e2."), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("1e-2."), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-1e-2."), false).IsString())
assert.True(t, inferWithIntAsFloat(FromDeferredType("1.2e3"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-1.2e3"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("1.2e-3"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-1.2e-3"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("1.e3"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-1.e3"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("1.e-3"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-1.e-3"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType(".2e3"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType("-.2e3"), false).IsFloat())
assert.True(t, inferWithIntAsFloat(FromDeferredType(".2e-3"), false).IsFloat())
assert.True(t, inferWithOctalAsInt(FromDeferredType("-.2e-3"), false).IsFloat())
}

func TestInferStringOnly(t *testing.T) {
assert.True(t, inferStringOnly(FromDeferredType(""), false).IsVoid())

assert.True(t, inferStringOnly(FromDeferredType("true"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("false"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("true"), true).IsString())
assert.True(t, inferStringOnly(FromDeferredType("false"), true).IsString())

assert.True(t, inferStringOnly(FromDeferredType("abc"), false).IsString())

assert.True(t, inferStringOnly(FromDeferredType("0123"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-0123"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("0377"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-0377"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("0923"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-0923"), false).IsString())

assert.True(t, inferStringOnly(FromDeferredType("123"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-123"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("0xff"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-0xff"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("0b1011"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-0b1011"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("0x7fffffffffffffff"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("0x8000000000000000"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("0xffffffffffffffff"), false).IsString())

assert.True(t, inferStringOnly(FromDeferredType("12_3"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-12_3"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("1_2.3_4"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-1_2.3_4"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("0xca_fe"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-0xca_fe"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("0b1011_1101"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-0b1011_1101"), false).IsString())

assert.True(t, inferStringOnly(FromDeferredType("."), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-."), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("123."), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-123."), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType(".123"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-.123"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("123.456"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-123.456"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("1e2."), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-1e2."), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("1e-2."), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-1e-2."), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("1.2e3"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-1.2e3"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("1.2e-3"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-1.2e-3"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("1.e3"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-1.e3"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("1.e-3"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-1.e-3"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType(".2e3"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-.2e3"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType(".2e-3"), false).IsString())
assert.True(t, inferStringOnly(FromDeferredType("-.2e-3"), false).IsString())
}