Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import io.github.dfa1.vortex.core.DType;
import io.github.dfa1.vortex.core.PType;
import io.github.dfa1.vortex.core.VortexException;
import io.github.dfa1.vortex.core.VortexFormat;
import io.github.dfa1.vortex.reader.array.ListArray;
import io.github.dfa1.vortex.reader.array.ListViewArray;
Expand All @@ -15,6 +16,7 @@
import java.util.List;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

/// Integration test: reads real Vortex files from the public S3 compatibility bucket
/// via HTTP Range requests and validates structure + data.
Expand All @@ -23,7 +25,7 @@
@Tag("integration")
class VortexHttpReaderIT {

private static final URI BASE = URI.create("https://vortex-compat-fixtures.s3.amazonaws.com/v0.72.0/arrays/");
private static final URI BASE = URI.create("https://vortex-compat-fixtures.s3.amazonaws.com/v0.75.0/arrays/");

private static final URI TPCH_LINEITEM = BASE.resolve("tpch_lineitem.compact.vortex");

Expand Down Expand Up @@ -78,13 +80,71 @@ void scan_forVortex_decodesAllRows() throws Exception {
assertThat(totalRows).isGreaterThan(0);
}

// vortex.masked / vortex.patched / vortex.variant: IDs known, decoders not yet implemented.
// No S3 fixture exists in v0.72.0 — enable these tests once fixtures are published.
// Smoke test: every published encoding fixture must open and decode all its rows.
// Catches upstream wire-format drift across the full encoding set in one place; the
// structural tests below (for/list/listview) assert shape for the trickier layouts.
// Excludes the multi-MB clickbench/tpch fixtures (covered by dedicated tests) to keep
// this fast over HTTP.
@ParameterizedTest
@ValueSource(strings = {
"alp.vortex", "alprd.vortex", "bitpacked.vortex", "booleans.vortex", "bytebool.vortex",
"chunked.vortex", "constant.vortex", "datetime.vortex", "datetimeparts.vortex",
"decimal.vortex", "decimal_byte_parts.vortex", "dict.vortex", "fixed_size_list.vortex",
"fsst.vortex", "null.vortex", "pco.vortex", "primitives.vortex", "rle.vortex",
"runend.vortex", "sequence.vortex", "sparse.vortex", "struct_nested.vortex",
"varbin.vortex", "varbinview.vortex", "zigzag.vortex"
// zstd.vortex omitted: dictionary-compressed, unsupported by the pure-Java decoder
// (see scan_zstdVortex_rejectsDictionaryCompression).
})
void scan_publishedFixture_decodesAllRows(String fixture) throws Exception {
// Given
URI uri = BASE.resolve(fixture);

// When
long totalRows = 0;
try (var sut = VortexHttpReader.open(uri);
var iter = sut.scan(ScanOptions.all())) {
while (iter.hasNext()) {
try (var c = iter.next()) {
totalRows += c.rowCount();
}
}
}

// Then
assertThat(totalRows).isGreaterThan(0);
}

// The published zstd.vortex fixture is dictionary-compressed; the pure-Java decoder has no
// Zstd dictionary support and must fail fast with a clear message rather than mis-decode.
// Tracked by https://github.com/dfa1/vortex-java/issues/104 (upstream airlift/aircompressor#119).
@Test
void scan_zstdVortex_rejectsDictionaryCompression() throws Exception {
// Given
URI uri = BASE.resolve("zstd.vortex");

// When / Then
try (var sut = VortexHttpReader.open(uri);
var iter = sut.scan(ScanOptions.all())) {
assertThatThrownBy(() -> {
while (iter.hasNext()) {
try (var c = iter.next()) {
c.rowCount();
}
}
})
.isInstanceOf(VortexException.class)
.hasMessageContaining("dictionary-compressed Zstd segments are not supported");
}
}

// vortex.masked / vortex.patched / vortex.variant: decoders implemented, but no S3 fixture
// is published (still absent at v0.75.0) — enable this test once fixtures exist upstream.

@Disabled("no S3 fixture in v0.72.0")
@Disabled("no S3 fixture through v0.75.0")
@ParameterizedTest
@ValueSource(strings = {"masked.vortex", "patched.vortex", "variant.vortex"})
void scan_unimplementedEncoding_decodesAllRows(String fixture) throws Exception {
void scan_unpublishedFixture_decodesAllRows(String fixture) throws Exception {
// Given
URI uri = BASE.resolve(fixture);

Expand Down