Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Port Symbol table aggregation improvements from DHE #3955

Merged
merged 8 commits into from
Jun 21, 2023
+444 −125
Merged

Port Symbol table aggregation improvements from DHE #3955

Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
89bdb65
Initial commit of SymbolTable aggregation port from DHE.
lbooker42 Jun 8, 2023
a8465e2
Minor change.
lbooker42 Jun 8, 2023
d6055a0
Rebased to main.
lbooker42 Jun 8, 2023
1b8af1c
Corrected missing import.
lbooker42 Jun 8, 2023
1cabc62
Addressed PR comments, needs more testing.
lbooker42 Jun 20, 2023
7289f97
Added multi-partition symbol table aggregation test.
lbooker42 Jun 20, 2023
6ac11e5
Added multi-partition symbol table aggregation test.
lbooker42 Jun 20, 2023
f9271f7
Modified multi-partition test and verified code coverage of new files.
lbooker42 Jun 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Added multi-partition symbol table aggregation test.
@lbooker42
lbooker42 committed Jun 20, 2023
commit 7289f97b3133c0722ebcabf7953ea2a2b81a3b42
43 changes: 43 additions & 0 deletions engine/table/src/test/java/io/deephaven/engine/table/impl/QueryTableAggregationTest.java
View file
Original file line number Diff line number Diff line change
@@ -36,8 +36,10 @@
import io.deephaven.engine.util.TableDiff;
import io.deephaven.engine.util.TableTools;
import io.deephaven.engine.util.systemicmarking.SystemicObjectTracker;
import io.deephaven.parquet.table.ParquetInstructions;
import io.deephaven.parquet.table.ParquetTableWriter;
import io.deephaven.parquet.table.ParquetTools;
import io.deephaven.parquet.table.layout.ParquetKeyValuePartitionedLayout;
import io.deephaven.qst.table.AggregateAllTable;
import io.deephaven.test.types.OutOfBandTest;
import io.deephaven.time.DateTimeUtils;
@@ -3833,6 +3835,47 @@ public void testPreserveEmptyNoKey() {
assertTableEquals(expectedEmpty, aggregated);
}

@Test
public void testMultiPartitionSymbolTableBy() throws IOException {
final File testRootFile = Files.createTempDirectory(QueryTableAggregationTest.class.getName()).toFile();
try {
final Table t1 = new InMemoryTable(
new String[] {"StringKeys", "GroupedInts"},
new Object[] {
new String[] {"key1", "key2", "key1", "key2", "key1", "key2", "key1", "key2", "key1"},
new short[] {1, 1, 2, 2, 2, 3, 3, 3, 3}
});
final Table t1_asc = t1.sort("StringKeys");
final Table t1_desc = t1.sortDescending("StringKeys");

ParquetTools.writeTable(t1, new File(testRootFile,
"Date=2021-07-20" + File.separator + "Num=100" + File.separator + "file1.parquet"));
ParquetTools.writeTable(t1_asc, new File(testRootFile,
"Date=2021-07-20" + File.separator + "Num=200" + File.separator + "file2.parquet"));
ParquetTools.writeTable(t1_desc, new File(testRootFile,
"Date=2021-07-21" + File.separator + "Num=300" + File.separator + "file3.parquet"));

final Table merged = TableTools.merge(
t1.updateView("Date=`2021-07-20`", "Num=100"),
t1_asc.updateView("Date=`2021-07-20`", "Num=200"),
t1_desc.updateView("Date=`2021-07-21`", "Num=300")).moveColumnsUp("Date", "Num");

final Table loaded = ParquetTools.readPartitionedTableInferSchema(
new ParquetKeyValuePartitionedLayout(testRootFile, 2), ParquetInstructions.EMPTY);

// make sure the sources are identical
assertTableEquals(merged, loaded);

final Table merged_summed = merged.aggBy(AggSum("GroupedInts"), "StringKeys");
final Table loaded_summed = loaded.aggBy(AggSum("GroupedInts"), "StringKeys");

TableTools.showWithRowSet(loaded_summed);
assertTableEquals(merged_summed, loaded_summed);
} finally {
FileUtils.deleteRecursively(testRootFile);
}
}

@Test
public void testSymbolTableBy() throws IOException {
diskBackedTestHarness((table) -> {