Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

mark int type explicitly as int64 #332

Merged
merged 2 commits into from
May 11, 2024
Merged

mark int type explicitly as int64 #332

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3c2339c
mark int type explicitly as int64
tjungblu Apr 24, 2024
d25a1ef
use 32bit truncation for parquet
tjungblu Apr 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion aggregates/count.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ var CountOverloads = []physical.AggregateDescriptor{
}

type Count struct {
count int
count int64
}

func NewCountPrototype() func() nodes.Aggregate {
Expand Down
2 changes: 1 addition & 1 deletion aggregates/sum.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ var SumOverloads = []physical.AggregateDescriptor{
}

type SumInt struct {
sum int
sum int64
}

func NewSumIntPrototype() func() nodes.Aggregate {
Expand Down
2 changes: 1 addition & 1 deletion cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,7 @@ octosql "SELECT * FROM plugins.plugins"`,

switch output {
case "live_table", "batch_table":
var limit *int
var limit *int64
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

let me know if that's OK for you, I think we could safely use the 32 bit here as before. Most of the PR here is to adjust for that type change.

if limitExpression != nil {
val, err := (*limitExpression).Evaluate(execCtx)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion datasources/csv/execution.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func (d *DatasourceExecuting) Run(ctx ExecutionContext, produce ProduceFn, metaS
if octosql.Int.Is(d.fields[i].Type) == octosql.TypeRelationIs {
integer, err := fastfloat.ParseInt64(str)
if err == nil {
values[i] = octosql.NewInt(int(integer))
values[i] = octosql.NewInt(integer)
continue
}
}
Expand Down
2 changes: 1 addition & 1 deletion datasources/lines/execution.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func (d *DatasourceExecuting) Run(ctx ExecutionContext, produce ProduceFn, metaS
})
}

line := 0
line := int64(0)
for sc.Scan() {
values := make([]octosql.Value, len(d.fields))
for i := range d.fields {
Expand Down
4 changes: 2 additions & 2 deletions datasources/parquet/reconstruct.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,9 +293,9 @@ func assignValue(dst *octosql.Value, src parquet.Value) error {
case parquet.Boolean:
*dst = octosql.NewBoolean(src.Boolean())
case parquet.Int32:
*dst = octosql.NewInt(int(src.Int32()))
*dst = octosql.NewInt(src.Int64())
Copy link
Owner

@cube2222 cube2222 Apr 25, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe this one might break (that is, if it's Int32 and you ask for Int64, the parquet library won't figure out that it can just upcast). Let's please leave it like it was, so int64(src.Int32()) now.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the code that I can see in parquet looks like that:

u64 uint64
... 
// Int32 returns v as a int32, assuming the underlying type is INT32.
func (v Value) Int32() int32 { return int32(v.u64) }

// Int64 returns v as a int64, assuming the underlying type is INT64.
func (v Value) Int64() int64 { return int64(v.u64) }

so we would simply cast the 64 bit integer to 32 bit and back

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've added your suggestion as a separate commit, in case you want to reconsider it, it's just a simple revert away :)

case parquet.Int64:
*dst = octosql.NewInt(int(src.Int64()))
*dst = octosql.NewInt(src.Int64())
case parquet.Int96:
*dst = octosql.NewString(src.Int96().String())
case parquet.Float:
Expand Down
2 changes: 1 addition & 1 deletion execution/nodes/limit.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func (m *Limit) Run(ctx ExecutionContext, produce ProduceFn, metaSend MetaSendFn

limitNodeID := ulid.MustNew(ulid.Now(), rand.Reader).String()

i := 0
i := int64(0)
if err := m.source.Run(ctx, func(produceCtx ProduceContext, record Record) error {
if err := produce(produceCtx, record); err != nil {
return fmt.Errorf("couldn't produce: %w", err)
Expand Down
8 changes: 4 additions & 4 deletions execution/nodes/order_sensitive_transform.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func (item *orderByItem) Less(than btree.Item) bool {
}

func (o *OrderSensitiveTransform) Run(execCtx ExecutionContext, produce ProduceFn, metaSend MetaSendFn) error {
var limit *int
var limit *int64
if o.limit != nil {
val, err := (*o.limit).Evaluate(execCtx)
if err != nil {
Expand Down Expand Up @@ -112,7 +112,7 @@ func (o *OrderSensitiveTransform) Run(execCtx ExecutionContext, produce ProduceF
} else {
recordCounts.Delete(itemTyped)
}
if limit != nil && o.noRetractionsPossible && recordCounts.Len() > *limit {
if limit != nil && o.noRetractionsPossible && int64(recordCounts.Len()) > *limit {
// This doesn't mean we'll always keep just the records that are needed, because tree nodes might have count > 1.
// That said, it's a good approximation, and we'll definitely not lose something that we need to have.
recordCounts.DeleteMax()
Expand All @@ -130,8 +130,8 @@ func (o *OrderSensitiveTransform) Run(execCtx ExecutionContext, produce ProduceF
return nil
}

func produceOrderByItems(ctx ProduceContext, recordCounts *btree.BTree, limit *int, produce ProduceFn) error {
i := 0
func produceOrderByItems(ctx ProduceContext, recordCounts *btree.BTree, limit *int64, produce ProduceFn) error {
i := int64(0)
var outErr error
recordCounts.Ascend(func(item btree.Item) bool {
if limit != nil && i >= *limit {
Expand Down
32 changes: 16 additions & 16 deletions functions/functions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,15 +304,15 @@ func FunctionMap() map[string]physical.FunctionDetails {
OutputType: octosql.String,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewString(strings.Repeat(values[0].Str, values[1].Int)), nil
return octosql.NewString(strings.Repeat(values[0].Str, int(values[1].Int))), nil
},
},
{
ArgumentTypes: []octosql.Type{octosql.Int, octosql.String},
OutputType: octosql.String,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewString(strings.Repeat(values[1].Str, values[0].Int)), nil
return octosql.NewString(strings.Repeat(values[1].Str, int(values[0].Int))), nil
},
},
},
Expand Down Expand Up @@ -718,7 +718,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
OutputType: octosql.String,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
if len(values[0].Str) <= values[1].Int {
if int64(len(values[0].Str)) <= values[1].Int {
return octosql.NewString(""), nil
}
return octosql.NewString(values[0].Str[values[1].Int:]), nil
Expand All @@ -729,12 +729,12 @@ func FunctionMap() map[string]physical.FunctionDetails {
OutputType: octosql.String,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
if len(values[0].Str) <= values[1].Int {
if int64(len(values[0].Str)) <= values[1].Int {
return octosql.NewString(""), nil
}
end := values[1].Int + values[2].Int
if end > len(values[0].Str) {
end = len(values[0].Str)
if end > int64(len(values[0].Str)) {
end = int64(len(values[0].Str))
}
return octosql.NewString(values[0].Str[values[1].Int:end]), nil
},
Expand Down Expand Up @@ -766,7 +766,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
if i == -1 {
return octosql.NewNull(), nil
}
return octosql.NewInt(i), nil
return octosql.NewInt(int64(i)), nil
},
},
},
Expand All @@ -779,7 +779,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
OutputType: octosql.Int,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewInt(len(values[0].Str)), nil
return octosql.NewInt(int64(len(values[0].Str))), nil
},
},
{
Expand All @@ -794,7 +794,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
},
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewInt(len(values[0].List)), nil
return octosql.NewInt(int64(len(values[0].List))), nil
},
},
{
Expand All @@ -809,7 +809,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
},
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewInt(len(values[0].Struct)), nil
return octosql.NewInt(int64(len(values[0].Struct))), nil
},
},
{
Expand All @@ -824,7 +824,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
},
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewInt(len(values[0].Tuple)), nil
return octosql.NewInt(int64(len(values[0].Tuple))), nil
},
},
},
Expand Down Expand Up @@ -891,7 +891,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
OutputType: octosql.Int,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewInt(int(values[0].Time.Unix())), nil
return octosql.NewInt(values[0].Time.Unix()), nil
},
},
},
Expand Down Expand Up @@ -927,15 +927,15 @@ func FunctionMap() map[string]physical.FunctionDetails {
OutputType: octosql.Int,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewInt(int(values[0].Float)), nil
return octosql.NewInt(int64(values[0].Float)), nil
},
},
{
ArgumentTypes: []octosql.Type{octosql.String},
OutputType: octosql.Int,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
n, err := strconv.Atoi(values[0].Str)
n, err := strconv.ParseInt(values[0].Str, 10, 64)
if err != nil {
log.Printf("couldn't parse string '%s' as int: %s", values[0].Str, err)
return octosql.NewNull(), nil
Expand All @@ -948,7 +948,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
OutputType: octosql.Int,
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
return octosql.NewInt(int(values[0].Duration)), nil
return octosql.NewInt(int64(values[0].Duration)), nil
},
},
},
Expand Down Expand Up @@ -1033,7 +1033,7 @@ func FunctionMap() map[string]physical.FunctionDetails {
},
Strict: true,
Function: func(values []octosql.Value) (octosql.Value, error) {
if values[1].Int >= len(values[0].List) {
if values[1].Int >= int64(len(values[0].List)) {
return octosql.NewNull(), nil
}
return values[0].List[values[1].Int], nil
Expand Down
4 changes: 2 additions & 2 deletions octosql/values.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ var ZeroValue = Value{}
// Value represents a single row value. The zero value of it is conveniently NULL.
type Value struct {
TypeID TypeID
Int int
Int int64
Float float64
Boolean bool
Str string
Expand All @@ -31,7 +31,7 @@ func NewNull() Value {
}
}

func NewInt(value int) Value {
func NewInt(value int64) Value {
return Value{
TypeID: TypeIDInt,
Int: value,
Expand Down
10 changes: 5 additions & 5 deletions outputs/batch/live_output.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,15 @@ type OutputPrinter struct {
source Node
keyExprs []Expression
directionMultipliers []int
limit *int
limit *int64
noRetractionsPossible bool

schema physical.Schema
format func(io.Writer) Format
live bool
}

func NewOutputPrinter(source Node, keyExprs []Expression, directionMultipliers []int, limit *int, noRetractionsPossible bool, schema physical.Schema, format func(io.Writer) Format, live bool) *OutputPrinter {
func NewOutputPrinter(source Node, keyExprs []Expression, directionMultipliers []int, limit *int64, noRetractionsPossible bool, schema physical.Schema, format func(io.Writer) Format, live bool) *OutputPrinter {
return &OutputPrinter{
source: source,
keyExprs: keyExprs,
Expand Down Expand Up @@ -89,7 +89,7 @@ func (o *OutputPrinter) Run(execCtx ExecutionContext) error {
format := o.format(&buf)
format.SetSchema(o.schema)

i := 0
i := int64(0)
recordCounts.Ascend(func(item btree.Item) bool {
itemTyped := item.(*outputItem)
for j := 0; j < itemTyped.Count; j++ {
Expand Down Expand Up @@ -157,7 +157,7 @@ func (o *OutputPrinter) Run(execCtx ExecutionContext) error {
if onlyZeroEventTimesSeen && !record.EventTime.IsZero() {
onlyZeroEventTimesSeen = false
}
if o.limit != nil && o.noRetractionsPossible && recordCounts.Len() > *o.limit {
if o.limit != nil && o.noRetractionsPossible && int64(recordCounts.Len()) > *o.limit {
// This doesn't mean we'll always keep just the records that are needed, because tree nodes might have count > 1.
// That said, it's a good approximation, and we'll definitely not lose something that we need to have.
recordCounts.DeleteMax()
Expand All @@ -183,7 +183,7 @@ func (o *OutputPrinter) Run(execCtx ExecutionContext) error {
var buf bytes.Buffer
format := o.format(&buf)
format.SetSchema(o.schema)
i := 0
i := int64(0)
recordCounts.Ascend(func(item btree.Item) bool {
itemTyped := item.(*outputItem)
for j := 0; j < itemTyped.Count; j++ {
Expand Down
2 changes: 1 addition & 1 deletion outputs/formats/json_format.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func ValueToJson(arena *fastjson.Arena, t octosql.Type, value octosql.Value) *fa
case octosql.TypeIDNull:
return arena.NewNull()
case octosql.TypeIDInt:
return arena.NewNumberInt(value.Int)
return arena.NewNumberInt(int(value.Int))
case octosql.TypeIDFloat:
return arena.NewNumberFloat64(value.Float)
case octosql.TypeIDBoolean:
Expand Down
2 changes: 1 addition & 1 deletion parser/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -629,7 +629,7 @@ func ParseExpression(expr sqlparser.Expr) (logical.Expression, error) {
case sqlparser.IntVal:
var i int64
i, err = strconv.ParseInt(string(expr.Val), 10, 64)
value = octosql.NewInt(int(i))
value = octosql.NewInt(i)
case sqlparser.FloatVal:
var val float64
val, err = strconv.ParseFloat(string(expr.Val), 64)
Expand Down
2 changes: 1 addition & 1 deletion plugins/internal/plugins/plugins.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ func (x *Value) ToNativeValue() octosql.Value {
switch octosql.TypeID(x.TypeId) {
case octosql.TypeIDNull:
case octosql.TypeIDInt:
out.Int = int(x.Int)
out.Int = x.Int
case octosql.TypeIDFloat:
out.Float = x.Float
case octosql.TypeIDBoolean:
Expand Down