dataset

type BuilderOptions struct {
	// PageSizeHint is the soft limit for the size of the page. Builders try to
	// fill pages as close to this size as possible, but the actual size may be
	// slightly larger or smaller.
	PageSizeHint int

	// Value is the value type of data to write.
	Value datasetmd.ValueType

	// Encoding is the encoding algorithm to use for values.
	Encoding datasetmd.EncodingType

	// Compression is the compression algorithm to use for values.
	Compression datasetmd.CompressionType

	// CompressionOptions holds optional configuration for compression.
	CompressionOptions CompressionOptions
}

type Column interface {
	// ColumnInfo returns the metadata for the Column.
	ColumnInfo() *ColumnInfo

	// ListPages returns the set of ordered pages in the column.
	ListPages(ctx context.Context) result.Seq[Page]
}

type ColumnBuilder struct {
	// contains filtered or unexported fields
}

type ColumnInfo struct {
	Name        string                    // Name of the column, if any.
	Type        datasetmd.ValueType       // Type of values in the column.
	Compression datasetmd.CompressionType // Compression used for the column.

	RowsCount        int // Total number of rows in the column.
	ValuesCount      int // Total number of non-NULL values in the column.
	CompressedSize   int // Total size of all pages in the column after compression.
	UncompressedSize int // Total size of all pages in the column before compression.

	Statistics *datasetmd.Statistics // Optional statistics for the column.
}

type CompressionOptions struct {
	// Zstd holds encoding options for Zstd compression. Only used for
	// [datasetmd.COMPRESSION_TYPE_ZSTD].
	Zstd []zstd.EOption
}

type Dataset interface {
	// ListColumns returns the set of [Column]s in the Dataset. The order of
	// Columns in the returned sequence must be consistent across calls.
	ListColumns(ctx context.Context) result.Seq[Column]

	// ListPages retrieves a set of [Pages] given a list of [Column]s.
	// Implementations of Dataset may use ListPages to optimize for batch reads.
	// The order of [Pages] in the returned sequence must match the order of the
	// columns argument.
	ListPages(ctx context.Context, columns []Column) result.Seq[Pages]

	// ReadPages returns the set of [PageData] for the specified slice of pages.
	// Implementations of Dataset may use ReadPages to optimize for batch reads.
	// The order of [PageData] in the returned sequence must match the order of
	// the pages argument.
	ReadPages(ctx context.Context, pages []Page) result.Seq[PageData]
}

type MemColumn struct {
	Info  ColumnInfo // Information about the column.
	Pages []*MemPage // The set of pages in the column.
}

type MemPage struct {
	Info PageInfo // Information about the page.
	Data PageData // Data for the page.
}

type Page interface {
	// PageInfo returns the metadata for the Page.
	PageInfo() *PageInfo

	// ReadPage returns the [PageData] for the Page.
	ReadPage(ctx context.Context) (PageData, error)
}

type PageData []byte

type PageInfo struct {
	UncompressedSize int    // UncompressedSize is the size of a page before compression.
	CompressedSize   int    // CompressedSize is the size of a page after compression.
	CRC32            uint32 // CRC32 checksum of the page after encoding and compression.
	RowCount         int    // RowCount is the number of rows in the page, including NULLs.
	ValuesCount      int    // ValuesCount is the number of non-NULL values in the page.

	Encoding datasetmd.EncodingType // Encoding used for values in the page.
	Stats    *datasetmd.Statistics  // Optional statistics for the page.
}

type Pages []Page

type Row struct {
	Index  int     // Index of the row in the dataset.
	Values []Value // Values for the row, one per [Column].
}

type Value struct {
	// contains filtered or unexported fields
}