From 7d4a13b17f01c1e0bf87712af4b19950a225a4b9 Mon Sep 17 00:00:00 2001 From: lemonviv Date: Mon, 20 Jan 2025 10:38:54 +0800 Subject: [PATCH 1/3] Add risefl git submodule --- .gitmodules | 4 ++++ third_party/README.md | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 563f8c955..40d4242ce 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,4 +9,8 @@ [submodule "third_party/served"] path = third_party/served url = https://github.com/meltwater/served.git + ignore = dirty +[submodule "third_party/risefl"] + path = third_party/risefl + url = https://github.com/nusdbsystem/risefl.git ignore = dirty \ No newline at end of file diff --git a/third_party/README.md b/third_party/README.md index 19d2a81df..d57e2b1d9 100644 --- a/third_party/README.md +++ b/third_party/README.md @@ -2,6 +2,6 @@ We use the following third-party dependencies in this project. * [SPDZ](https://github.com/data61/MP-SPDZ): providing additive secret sharing based MPC. - * [libhcs](https://github.com/tiehuis/libhcs): providing Paillier homomorphic - encryption operations. - * [SINGA](https://singa.apache.org/): providing local neural networks optimizer in horizontal FL. \ No newline at end of file + * [libhcs](https://github.com/tiehuis/libhcs): providing Paillier homomorphic encryption operations. + * [served](https://github.com/meltwater/served.git): providing inference service. + * [risefl](https://github.com/nusdbsystem/risefl.git): providing input integrity check with zero-knowledge proofs. \ No newline at end of file From bc1f0cf941748d8261f64b7bc2818ed478406c5b Mon Sep 17 00:00:00 2001 From: lemonviv Date: Mon, 20 Jan 2025 10:43:25 +0800 Subject: [PATCH 2/3] Update dockerfile --- deployment/ubuntu18.04-falcon.Dockerfile | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/deployment/ubuntu18.04-falcon.Dockerfile b/deployment/ubuntu18.04-falcon.Dockerfile index 7aeb91c03..9e1eaf5b0 100644 --- a/deployment/ubuntu18.04-falcon.Dockerfile +++ b/deployment/ubuntu18.04-falcon.Dockerfile @@ -138,6 +138,9 @@ RUN apt-get update && apt-get upgrade -y && \ python3-pip \ libgoogle-glog-dev \ libgtest-dev \ + libgmp3-dev \ + swig \ + libtbb-dev \ && \ pip3 install requests && \ apt-get clean && \ @@ -176,7 +179,7 @@ RUN echo "update repo for building new image" WORKDIR /opt RUN echo yes|git clone git@github.com:lemonviv/falcon.git && \ cd falcon && \ - git checkout dev && \ + git checkout hfl && \ cd third_party/ && \ git submodule update --init --recursive @@ -211,6 +214,20 @@ RUN cd third_party/served && \ make && \ make install +# Install libntl library +RUN cd /root/temp && \ + wget https://libntl.org/ntl-11.5.1.tar.gz && \ + tar -xzvf ntl-11.5.1.tar.gz && \ + cd ntl-11.5.1/src/ && \ + ./configure && \ + make && make check && \ + make install + +# Install risefl library +WORKDIR /opt/falcon +RUN cd third_party/risefl && \ + bash make.sh + # generate protobuf messages WORKDIR /opt/falcon RUN cd /opt/falcon/src/executor/include/proto && \ From 258624f8ca0ba2da851ef1af0e3faa282ba7fc03 Mon Sep 17 00:00:00 2001 From: lemonviv Date: Mon, 20 Jan 2025 10:48:27 +0800 Subject: [PATCH 3/3] Update dockerfile --- deployment/ubuntu-18.04-falcon-hfl.Dockerfile | 344 ++++++++++++++++++ deployment/ubuntu18.04-falcon.Dockerfile | 19 +- 2 files changed, 345 insertions(+), 18 deletions(-) create mode 100644 deployment/ubuntu-18.04-falcon-hfl.Dockerfile diff --git a/deployment/ubuntu-18.04-falcon-hfl.Dockerfile b/deployment/ubuntu-18.04-falcon-hfl.Dockerfile new file mode 100644 index 000000000..83bcb4273 --- /dev/null +++ b/deployment/ubuntu-18.04-falcon-hfl.Dockerfile @@ -0,0 +1,344 @@ +### the dependencies image +FROM ubuntu:18.04 as dependencies + +LABEL maintainer="Yuncheng Wu " + +RUN apt-get update && apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + build-essential \ + libprotobuf-c0-dev \ + libprotobuf-dev \ + protobuf-compiler \ + python \ + python3 \ + python3-pip \ + libgoogle-glog-dev \ + libgtest-dev \ + pkg-config \ + wget \ + automake \ + ca-certificates \ + gdb \ + git \ + unzip \ + libjsoncpp-dev \ + liblog4cpp5-dev \ + libssl-dev \ + libtool \ + && \ + pip3 install requests && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Install +RUN dpkg-reconfigure dash + +# Upgrade cmake version to 3.19.7 +RUN mkdir /root/temp && \ + cd ~/temp && \ + wget https://cmake.org/files/v3.19/cmake-3.19.7.tar.gz && \ + tar -xzvf cmake-3.19.7.tar.gz && \ + cd cmake-3.19.7/ && \ + ./bootstrap && \ + make -j$(nproc) && \ + make install && \ + hash -r && \ + cmake --version + +# Install glog library +WORKDIR /root/temp +RUN git clone https://github.com/google/glog.git && \ + cd glog && \ + cmake -H. -Bbuild -G "Unix Makefiles" && \ + cmake --build build + +# Ln gtest library +RUN cd /usr/src/googletest/googletest && \ + mkdir build && \ + cd build && \ + cmake .. && \ + make && \ + cp libgtest* /usr/lib/ && \ + cd .. && \ + rm -rf build && \ + mkdir /usr/local/lib/googletest && \ + ln -s /usr/lib/libgtest.a /usr/local/lib/googletest/libgtest.a && \ + ln -s /usr/lib/libgtest_main.a /usr/local/lib/googletest/libgtest_main.a + +# Install Go 1.14 +WORKDIR /root/temp/ +RUN wget -q https://golang.org/dl/go1.14.13.linux-amd64.tar.gz -O go114.tar.gz && \ + tar xzf go114.tar.gz -C /usr/local + +# Replace protoc version +WORKDIR /root/temp/ +RUN git clone -b 'v3.14.0' --single-branch --depth 1 https://github.com/protocolbuffers/protobuf.git && \ + cd protobuf && \ + git submodule update --init --recursive && \ + ./autogen.sh && \ + ./configure && \ + make && \ + make check && \ + make install && \ + ldconfig && \ + protoc --version && \ + # the following add the protoc path to cmake config + cd cmake && \ + mkdir build && \ + cd build && \ + cmake .. && \ + make && make install + + +### the build image that download the code and build +FROM ubuntu:18.04 as build + +COPY --from=dependencies /usr/local /usr/local +COPY --from=dependencies /usr/bin /usr/bin +COPY --from=dependencies /usr/lib /usr/lib + +RUN apt-get update && apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + git \ + sqlite3 \ + curl \ + sudo \ + unzip \ + vim \ + wget \ + zip \ + ssh \ + libgmp-dev \ + libboost-dev \ + libboost-all-dev \ + libboost-system-dev \ + libboost-thread-dev \ + libcurl4-openssl-dev \ + build-essential \ + ca-certificates \ + gdb \ + git \ + unzip \ + libjsoncpp-dev \ + liblog4cpp5-dev \ + libssl-dev \ + libtool \ + m4 \ + texinfo \ + yasm \ + automake \ + libsodium-dev \ + libcrypto++-dev \ + libcrypto++-doc \ + libcrypto++-utils \ + autoconf \ + pkg-config \ + python \ + python3 \ + python3-pip \ + libgoogle-glog-dev \ + libgtest-dev \ + libgmp3-dev \ + swig \ + libtbb-dev \ + && \ + pip3 install requests && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +#Accept input argument +ARG SSH_PRIVATE_KEY + +#Pass the content of the private key into the container +RUN mkdir /root/.ssh/ +RUN echo "${SSH_PRIVATE_KEY}" > /root/.ssh/id_rsa + +#Github requires a private key with strict permission settings +RUN chmod 600 /root/.ssh/id_rsa + +#Add Github to known hosts +RUN touch /root/.ssh/config +RUN echo Host github.com > /root/.ssh/config +RUN echo Hostname ssh.github.com >> /root/.ssh/config +RUN echo Port 443 >> /root/.ssh/config +RUN echo StrictHostKeyChecking no >> /root/.ssh/config + +RUN touch /root/.ssh/known_hosts +RUN ssh-keyscan github.com >> /root/.ssh/known_hosts +RUN git config --global http.sslVerify false + +#RUN touch /root/.ssh/config +#RUN echo Host github.com > /root/.ssh/config +#RUN echo Hostname ssh.github.com >> /root/.ssh/config +#RUN echo Port 443 >> /root/.ssh/config + +RUN cat /root/.ssh/config + +# Clone Falcon and init submodules +RUN echo "update repo for building new image" +WORKDIR /opt +RUN echo yes|git clone git@github.com:lemonviv/falcon.git && \ + cd falcon && \ + git checkout hfl && \ + cd third_party/ && \ + git submodule update --init --recursive + +# Install third_party threshold partially homomorphic encryption library +WORKDIR /opt/falcon +RUN cd third_party/libhcs && \ + cmake . && \ + make && \ + make install + +# Install third_party MP-SPDZ library +WORKDIR /opt/falcon +RUN cd third_party/MP-SPDZ && \ + mv CONFIG.mine.prod CONFIG.mine && \ + mv Math/Setup.h.prod Math/Setup.h && \ + make mpir && \ + bash fast-make.sh && \ + Scripts/setup-clients.sh 3 && \ + ./compile.py Programs/Source/logistic_regression.mpc && \ + ./compile.py Programs/Source/linear_regression.mpc && \ + ./compile.py Programs/Source/lime.mpc && \ + ./compile.py Programs/Source/vfl_decision_tree.mpc && \ + ./compile.py Programs/Source/mlp.mpc && \ + ln -s /opt/falcon/third_party/MP-SPDZ/local/lib/libmpir* /usr/local/lib/ + +# Install served library +WORKDIR /opt/falcon +RUN cd third_party/served && \ + mkdir cmake.build && \ + cd cmake.build && \ + cmake ../ && \ + make && \ + make install + +# Install libsodium 1.0.18 +RUN mkdir /root/temp && \ + cd /root/temp && \ + wget https://download.libsodium.org/libsodium/releases/libsodium-1.0.18-stable.tar.gz && \ + tar -xzvf libsodium-1.0.18-stable.tar.gz && \ + cd libsodium-stable/ && \ + ./configure && \ + make && make check && \ + make install \ + +# Install libntl library +RUN cd /root/temp && \ + wget https://libntl.org/ntl-11.5.1.tar.gz && \ + tar -xzvf ntl-11.5.1.tar.gz && \ + cd ntl-11.5.1/src/ && \ + ./configure && \ + make && make check && \ + make install + +# Install risefl library +WORKDIR /opt/falcon +RUN cd third_party/risefl && \ + bash make.sh + +# generate protobuf messages +WORKDIR /opt/falcon +RUN cd /opt/falcon/src/executor/include/proto && \ + SRC_DIR=v0/ && \ + DST_DIR=../message && \ + protoc -I=$SRC_DIR --cpp_out=$DST_DIR $SRC_DIR/*.proto + +# build the falcon executor +WORKDIR /opt/falcon +RUN git branch +RUN git pull origin dev && git log +RUN export PATH="$PATH:$HOME/.local/bin" && \ + mkdir build && \ + cmake -Bbuild -H. && \ + cd build/ && \ + make + + +### the final release image +FROM ubuntu:18.04 as release + +# copy data from the build image +COPY --from=build /opt/falcon /opt/falcon +COPY --from=build /usr/local /usr/local +COPY --from=build /usr/bin /usr/bin +COPY --from=build /usr/lib /usr/lib +COPY --from=build /root/.ssh /root/.ssh + +RUN apt-get update && apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + git \ + build-essential \ + ca-certificates \ + libboost-dev \ + libboost-all-dev \ + libboost-system-dev \ + libboost-thread-dev \ + libgoogle-glog-dev \ + libgtest-dev \ + libsodium-dev \ + libcurl4-openssl-dev \ + libssl-dev \ + sudo && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +############################################################# +# adjust the following according to the developing progress +############################################################# +WORKDIR /opt/falcon/third_party/MP-SPDZ +RUN git fetch origin && \ + git checkout master && \ + git log --oneline -2 && \ + ./compile.py Programs/Source/lime.mpc + +# 1. pull latest code +WORKDIR /opt/falcon +RUN git fetch origin && \ + git checkout dev && \ + git pull origin dev + +# 2. Set environment variables and pre-compile falcon_coordinator +ENV GOROOT /usr/local/go +ENV GOPATH /gopath +ENV PATH $GOROOT/bin:$GOPATH/bin:$PATH +ENV PATH /root/.local/bin:$PATH +WORKDIR /opt/falcon/src/falcon_platform +RUN bash make_platform.sh + +# ARG CACHEBUST=1 is to force re-execute the following CMDs at each updates. +ARG CACHEBUST=1 + +# 1. pull latest code +WORKDIR /opt/falcon +RUN git fetch origin && \ + git pull origin dev + +# 2. Set environment variables and pre-compile falcon_coordinator +# ENV GOROOT /usr/local/go +# ENV GOPATH /gopath +# ENV PATH $GOROOT/bin:$GOPATH/bin:$PATH +# ENV PATH /root/.local/bin:$PATH +# WORKDIR /opt/falcon/src/falcon_platform +# RUN bash make_platform.sh + +# 3. update mpc data and code +# Client-side handshake with P0 failed. Make sure we have the necessary certificate (Player-Data/P0.pem in the default configuration), and run `c_rehash ` on its location. +# The certificates should be the same on every host. Also make sure that it's still valid. Certificates generated with `Scripts/setup-ssl.sh` expire after a month. +WORKDIR /opt/falcon/third_party/MP-SPDZ +RUN Scripts/setup-online.sh 3 128 128 && \ + Scripts/setup-clients.sh 3 && \ + Scripts/setup-ssl.sh 3 128 128 && \ + c_rehash Player-Data/ && \ + git pull && \ + git log --oneline -2 && \ + ./compile.py Programs/Source/lime.mpc + +# 4. pre-compile falcon +WORKDIR /opt/falcon +RUN bash make.sh + +# 5. define the entry +WORKDIR /opt/falcon +CMD ["bash", "deployment/docker_cmd.sh"] diff --git a/deployment/ubuntu18.04-falcon.Dockerfile b/deployment/ubuntu18.04-falcon.Dockerfile index 9e1eaf5b0..7aeb91c03 100644 --- a/deployment/ubuntu18.04-falcon.Dockerfile +++ b/deployment/ubuntu18.04-falcon.Dockerfile @@ -138,9 +138,6 @@ RUN apt-get update && apt-get upgrade -y && \ python3-pip \ libgoogle-glog-dev \ libgtest-dev \ - libgmp3-dev \ - swig \ - libtbb-dev \ && \ pip3 install requests && \ apt-get clean && \ @@ -179,7 +176,7 @@ RUN echo "update repo for building new image" WORKDIR /opt RUN echo yes|git clone git@github.com:lemonviv/falcon.git && \ cd falcon && \ - git checkout hfl && \ + git checkout dev && \ cd third_party/ && \ git submodule update --init --recursive @@ -214,20 +211,6 @@ RUN cd third_party/served && \ make && \ make install -# Install libntl library -RUN cd /root/temp && \ - wget https://libntl.org/ntl-11.5.1.tar.gz && \ - tar -xzvf ntl-11.5.1.tar.gz && \ - cd ntl-11.5.1/src/ && \ - ./configure && \ - make && make check && \ - make install - -# Install risefl library -WORKDIR /opt/falcon -RUN cd third_party/risefl && \ - bash make.sh - # generate protobuf messages WORKDIR /opt/falcon RUN cd /opt/falcon/src/executor/include/proto && \