FastLanes: Next-Gen Big Data File Format

3 months ago 2

Like Parquet, but with 40% better compression and 40× faster decoding.

[dependencies] fls-rs = { path = "./rust" }
@article{afroozeh2023fastlanes, author = {Afroozeh, Azim and Boncz, Peter}, title = {The FastLanes Compression Layout: Decoding > 100 Billion Integers per Second with Scalar Code}, journal = {Proceedings of the VLDB Endowment}, volume = {16}, number = {9}, pages = {2132--2144}, month = may, year = {2023}, publisher = {VLDB Endowment} } @inproceedings{afroozeh2024accelerating, author = {Afroozeh, Azim and Felius, Lotte and Boncz, Peter}, title = {Accelerating GPU Data Processing Using FastLanes Compression}, booktitle = {DaMoN ’24: Proceedings of the 20th International Workshop on Data Management on New Hardware}, pages = {1--11}, month = jun, year = {2024}, organization = {ACM}, doi = {10.1145/3662010.3663450} } @inproceedings{afroozeh2024alp, author = {Afroozeh, Azim and Kuffó, Leonardo and Boncz, Peter}, title = {ALP: Adaptive Lossless Floating-Point Compression}, booktitle = {SIGMOD ’24: Proceedings of the 2024 ACM SIGMOD International Conference on Management of Data}, pages = {1--13}, month = jun, year = {2024}, organization = {ACM}, doi = {10.1145/3626717} } @inproceedings{hepke2025galp, author = {Hepkema, Sven Hielke and Afroozeh, Azim and Felius, Charlotte and Boncz, Peter and Manegold, Stefan}, title = {G‑ALP: Rethinking Light‑weight Encodings for GPUs}, booktitle = {DaMoN ’25: Proceedings of the 21st International Workshop on Data Management on New Hardware}, pages = {11:1--11:10}, month = jul, year = {2025}, organization = {ACM}, url = {https://dl.acm.org/doi/pdf/10.1145/3736227.3736242} }

Come discuss FastLanes, share feedback, and help shape the future of data formats on Discord: Join Our Discord

Read Entire Article