Bring LZ4 support for hardnested tables

Testing perfs on an old 8-core laptop:
BZ2   :   2 Mb, average decompression time: 5514 ms
LZ4 -9:   9 Mb, average decompression time:  837 ms
RAW   : 704 Mb, average decompression time:  368 ms

Other LZ4 options don't help: -1 is slower and bigger, --favor-decSpeed does not do anything

To enable:

apt install liblz4-dev
cd client/resources/hardnested_tables
for i in *bz2; do bunzip2 $i; lz4 -9 --rm ${i%.bz2}; done
This commit is contained in:
Philippe Teuwen 2023-07-15 16:52:48 +02:00
commit 83e083100b
717 changed files with 220 additions and 79 deletions

View file

@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && \
apt-get upgrade -y && \
apt-get dist-upgrade -y && \
apt-get install -y --no-install-recommends git ca-certificates build-essential cmake pkg-config libreadline-dev gcc-arm-none-eabi libnewlib-dev libbz2-dev libbluetooth-dev libpython3-dev libssl-dev sudo && \
apt-get install -y --no-install-recommends git ca-certificates build-essential cmake pkg-config libreadline-dev gcc-arm-none-eabi libnewlib-dev libbz2-dev liblz4-dev libbluetooth-dev libpython3-dev libssl-dev sudo && \
apt-get clean
RUN apt-get install -y python3-minimal && \

View file

@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && \
apt-get upgrade -y && \
apt-get dist-upgrade -y && \
apt-get install -y --no-install-recommends git ca-certificates build-essential cmake pkg-config libreadline-dev gcc-arm-none-eabi libnewlib-dev libbz2-dev libbluetooth-dev libpython3-dev libssl-dev sudo && \
apt-get install -y --no-install-recommends git ca-certificates build-essential cmake pkg-config libreadline-dev gcc-arm-none-eabi libnewlib-dev libbz2-dev liblz4-dev libbluetooth-dev libpython3-dev libssl-dev sudo && \
apt-get clean
RUN apt-get install -y python3-minimal && \

View file

@ -2,7 +2,7 @@ FROM opensuse/leap
ENV LANG C
# libqt5-qtbase-devel skipped
RUN zypper --non-interactive install --no-recommends shadow sudo git patterns-devel-base-devel_basis gcc-c++ readline-devel libbz2-devel bluez-devel python3-devel libopenssl-devel
RUN zypper --non-interactive install --no-recommends shadow sudo git patterns-devel-base-devel_basis gcc-c++ readline-devel libbz2-devel liblz4-devel bluez-devel python3-devel libopenssl-devel
RUN zypper addrepo https://download.opensuse.org/repositories/home:wkazubski/15.4/home:wkazubski.repo && \
zypper --gpg-auto-import-keys refresh && \

View file

@ -2,7 +2,7 @@ FROM opensuse/tumbleweed
ENV LANG C
# libqt5-qtbase-devel skipped
RUN zypper --non-interactive install --no-recommends shadow sudo git patterns-devel-base-devel_basis gcc-c++ readline-devel libbz2-devel bluez-devel python3-devel libopenssl-devel cross-arm-none-gcc12 cross-arm-none-newlib-devel
RUN zypper --non-interactive install --no-recommends shadow sudo git patterns-devel-base-devel_basis gcc-c++ readline-devel libbz2-devel liblz4-devel bluez-devel python3-devel libopenssl-devel cross-arm-none-gcc12 cross-arm-none-newlib-devel
#RUN zypper addrepo https://download.opensuse.org/repositories/home:wkazubski/openSUSE_Tumbleweed/home:wkazubski.repo && \
# zypper --gpg-auto-import-keys refresh && \

View file

@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && \
apt-get upgrade -y && \
apt-get dist-upgrade -y && \
apt-get install -y --no-install-recommends git ca-certificates build-essential cmake pkg-config libreadline-dev gcc-arm-none-eabi libnewlib-dev libbz2-dev libbluetooth-dev libpython3-dev libssl-dev sudo && \
apt-get install -y --no-install-recommends git ca-certificates build-essential cmake pkg-config libreadline-dev gcc-arm-none-eabi libnewlib-dev libbz2-dev liblz4-dev libbluetooth-dev libpython3-dev libssl-dev sudo && \
apt-get clean
RUN apt-get install -y python3-minimal && \

View file

@ -7,7 +7,7 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && \
apt-get upgrade -y && \
apt-get dist-upgrade -y && \
apt-get install -y --no-install-recommends git ca-certificates build-essential cmake pkg-config libreadline-dev gcc-arm-none-eabi libnewlib-dev libbz2-dev libbluetooth-dev libssl-dev sudo && \
apt-get install -y --no-install-recommends git ca-certificates build-essential cmake pkg-config libreadline-dev gcc-arm-none-eabi libnewlib-dev libbz2-dev liblz4-dev libbluetooth-dev libssl-dev sudo && \
apt-get clean
RUN apt-get install -y python3-minimal && \

View file

@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && \
apt-get upgrade -y && \
apt-get dist-upgrade -y && \
apt-get install -y --no-install-recommends git ca-certificates build-essential cmake pkg-config libreadline-dev gcc-arm-none-eabi libnewlib-dev libbz2-dev libbluetooth-dev libpython3-dev libssl-dev sudo && \
apt-get install -y --no-install-recommends git ca-certificates build-essential cmake pkg-config libreadline-dev gcc-arm-none-eabi libnewlib-dev libbz2-dev liblz4-dev libbluetooth-dev libpython3-dev libssl-dev sudo && \
apt-get clean
RUN apt-get install -y python3-minimal && \

View file

@ -6,7 +6,7 @@ ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && \
apt-get upgrade -y && \
apt-get dist-upgrade -y && \
apt-get install -y --no-install-recommends git ca-certificates build-essential cmake pkg-config libreadline-dev gcc-arm-none-eabi libnewlib-dev libbz2-dev libbluetooth-dev libpython3-dev libssl-dev sudo && \
apt-get install -y --no-install-recommends git ca-certificates build-essential cmake pkg-config libreadline-dev gcc-arm-none-eabi libnewlib-dev libbz2-dev liblz4-dev libbluetooth-dev libpython3-dev libssl-dev sudo && \
apt-get clean
RUN apt-get install -y python3-minimal && \