Skip to content

Commit c04e446

Browse files
lgqsslgqss
authored
Merge 2024-05-30 internal (#252)
* merge_2024_05_30_internal * merge_2024_05_30_internal2 * fix compile * fix cmake compile * fix cmake * fix compile * fix cmale compile * fix bazel compile * fix cmake compile * fix cmake compile * fix cmake arrow compile * fix cmake arrow compile * fix cmake arrow compile * fix cmake arrow compile * fix cmake arrow compile * fix cmake arrow compile * fix cmake arrow compile * fix cmake arrow compile * fix cmake arrow compile * fix cmake arrow compile * fix cmake arrow compile * fix cmake arrow compile
1 parent 8e9279a commit c04e446

File tree

306 files changed

+32594
-5186
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

306 files changed

+32594
-5186
lines changed

‎.github/workflows/bazel.yml ‎.github/workflows/bazel_debug.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
name: Bazel Compile
1+
name: Bazel Debug
22

33
on:
4-
[push,pull_request]
4+
[workflow_dispatch]
55

66
jobs:
77
build:

‎.github/workflows/cmake.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,14 @@ jobs:
77
build:
88
runs-on: ubuntu-20.04
99
steps:
10-
- uses: actions/checkout@v2
10+
- uses: actions/checkout@v3
1111
- name: Cache cmake modules
12-
uses: actions/cache@v2
12+
uses: actions/cache@v4
1313
with:
14-
key: cmake-buildenv-third-party
14+
key: cmake-buildenv-third-party2
1515
path: |
1616
buildenv/third-party
1717
- name: Install Dependencies
1818
run: sudo apt-get update && sudo apt-get install -y flex bison libssl-dev autoconf g++ libtool make cmake libz-dev locate gfortran
1919
- name: Build
20-
run: mkdir -p buildenv && cd buildenv && cmake -DWITH_BAIKAL_CLIENT=OFF .. && make
20+
run: mkdir -p buildenv && cd buildenv && cmake -DWITH_BAIKAL_CLIENT=OFF .. && make -j2

‎.github/workflows/cmake_debug.yml

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
name: Cmake Debug
2+
3+
on:
4+
[workflow_dispatch]
5+
6+
jobs:
7+
build:
8+
runs-on: ubuntu-20.04
9+
steps:
10+
- uses: actions/checkout@v3
11+
- name: Cache cmake modules
12+
uses: actions/cache@v4
13+
with:
14+
key: cmake-buildenv-third-party2
15+
path: |
16+
buildenv/third-party
17+
- name: Install Dependencies
18+
run: sudo apt-get update && sudo apt-get install -y flex bison libssl-dev autoconf g++ libtool make cmake libz-dev locate gfortran
19+
- name: Build
20+
run: mkdir -p buildenv && cd buildenv && cmake -DWITH_BAIKAL_CLIENT=OFF .. && make -j2
21+
- name: Start tmate session
22+
if: failure() # 只有失败时运行
23+
uses: mxschmitt/action-tmate@v3

‎CMakeLists.txt

+17-14
Original file line numberDiff line numberDiff line change
@@ -108,18 +108,6 @@ else ()
108108
include(rapidjson)
109109
endif ()
110110

111-
#apache arrow
112-
if (WITH_SYSTEM_LIBS)
113-
find_path(ARROW_INCLUDE_DIR NAMES arrow/api.h)
114-
find_library(ARROW_LIBRARIES NAMES arrow)
115-
if ((NOT ARROW_INCLUDE_DIR) OR (NOT ARROW_LIBRARIES))
116-
message(FATAL_ERROR "Fail to find arrow")
117-
endif ()
118-
ADD_LIBRARY(arrow SHARED IMPORTED GLOBAL)
119-
else ()
120-
include(arrow)
121-
endif ()
122-
123111
#openblas
124112
if (WITH_SYSTEM_LIBS)
125113
find_path(OPENBLAS_INCLUDE_DIR NAMES openblas/api.h)
@@ -308,6 +296,18 @@ if (WITH_GPERF)
308296
endif ()
309297
endif ()
310298

299+
#apache arrow
300+
if (WITH_SYSTEM_LIBS)
301+
find_path(ARROW_INCLUDE_DIR NAMES arrow/api.h)
302+
find_library(ARROW_LIBRARIES NAMES arrow)
303+
if ((NOT ARROW_INCLUDE_DIR) OR (NOT ARROW_LIBRARIES))
304+
message(FATAL_ERROR "Fail to find arrow")
305+
endif ()
306+
ADD_LIBRARY(arrow SHARED IMPORTED GLOBAL)
307+
else ()
308+
include(arrow)
309+
endif ()
310+
311311
message(CUR_DIR : ${CMAKE_CURRENT_BINARY_DIR}, SRC_DIR : ${CMAKE_SOURCE_DIR})
312312

313313
file(GLOB PROTO_FILES ${CMAKE_SOURCE_DIR}/proto/*.proto)
@@ -387,6 +387,11 @@ endif ()
387387

388388
SET(DEP_LIB
389389
${DEP_LIB}
390+
${ARROW_ACERO_LIB}
391+
${ARROW_LIBRARIES}
392+
${ARROW_ACERO_LIB}
393+
${ARROW_PARQUET_LIB}
394+
${ARROW_BUNDLED_DEP_LIB}
390395
${PROTOBUF_LIBRARIES}
391396
${PROTOBUF_PROTOC_LIBRARY}
392397
${GFLAGS_LIBRARIES}
@@ -396,12 +401,10 @@ SET(DEP_LIB
396401
${LZ4_LIBRARIES}
397402
${LIBURING_LIBRARIES}
398403
${RE2_LIBRARIES}
399-
${ARROW_LIBRARIES}
400404
${FAISS_LIBRARIES}
401405
${OPENBLAS_LIBRARIES}
402406
${CROARING_LIBRARIES}
403407
${Boost_LIBRARIES}
404-
# ${BZ2_LIBRARIES}
405408
)
406409

407410
if (WITH_GPERF)

‎WORKSPACE

+2-2
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ bind(
2727
new_http_archive(
2828
name = "com_github_apache_arrow",
2929
build_file = "third-party/com_github_apache_arrow/BUILD",
30-
strip_prefix = "arrow-apache-arrow-0.17.1",
31-
url = "https://github.com/apache/arrow/archive/apache-arrow-0.17.1.tar.gz",
30+
strip_prefix = "arrow-16.1.0",
31+
url = "https://github.com/baikalgroup/arrow/archive/refs/tags/16.1.0.tar.gz",
3232
)
3333

3434
bind(

‎bazel/baikaldb.bzl

100644100755
File mode changed.

‎cmake/arrow.cmake

+22-4
Original file line numberDiff line numberDiff line change
@@ -18,25 +18,43 @@ SET(ARROW_SOURCES_DIR ${THIRD_PARTY_PATH}/arrow)
1818
SET(ARROW_INSTALL_DIR ${THIRD_PARTY_PATH}/install/arrow)
1919
SET(ARROW_INCLUDE_DIR "${ARROW_INSTALL_DIR}/include" CACHE PATH "arrow include directory." FORCE)
2020
SET(ARROW_LIBRARIES "${ARROW_INSTALL_DIR}/lib/libarrow.a" CACHE FILEPATH "arrow library." FORCE)
21+
SET(ARROW_PARQUET_LIB "${ARROW_INSTALL_DIR}/lib/libparquet.a" CACHE FILEPATH "arrow parquet." FORCE)
22+
SET(ARROW_ACERO_LIB "${ARROW_INSTALL_DIR}/lib/libarrow_acero.a" CACHE FILEPATH "arrow acero." FORCE)
23+
SET(ARROW_BUNDLED_DEP_LIB "${ARROW_INSTALL_DIR}/lib/libarrow_bundled_dependencies.a" CACHE FILEPATH "arrow dependencies." FORCE)
2124

2225
FILE(WRITE ${ARROW_SOURCES_DIR}/src/build.sh
23-
"cd cpp && cmake -DCMAKE_BUILD_TYPE=release -DARROW_JEMALLOC=OFF -DARROW_BUILD_SHARED=OFF && make -j${NUM_OF_PROCESSOR}"
26+
"cd cpp && cmake -DCMAKE_BUILD_TYPE=release -DARROW_JEMALLOC=OFF -DARROW_BUILD_SHARED=OFF -DARROW_PARQUET=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_SNAPPY=ON -DARROW_COMPUTE=ON -DARROW_ACERO=ON -DARROW_FILESYSTEM=ON -DARROW_JSON=ON -DARROW_PARQUET=ON -DARROW_BUILD_TESTS=OFF -DARROW_BUILD_STATIC=ON && make -j2"
2427
)
2528

2629
ExternalProject_Add(
2730
extern_arrow
2831
${EXTERNAL_PROJECT_LOG_ARGS}
2932
# GIT_REPOSITORY "https://github.com/apache/arrow.git"
3033
# GIT_TAG "apache-arrow-0.17.1"
31-
URL "https://github.com/apache/arrow/archive/apache-arrow-0.17.1.tar.gz"
34+
URL "https://github.com/baikalgroup/arrow/archive/refs/tags/16.1.0.tar.gz"
3235
PREFIX ${ARROW_SOURCES_DIR}
3336
UPDATE_COMMAND ""
3437
CONFIGURE_COMMAND ""
3538
BUILD_IN_SOURCE 1
3639
BUILD_COMMAND mv ../build.sh . COMMAND sh build.sh
37-
INSTALL_COMMAND mkdir -p ${ARROW_INSTALL_DIR}/lib/ COMMAND cp ${ARROW_SOURCES_DIR}/src/extern_arrow/cpp/build/release/libarrow.a ${ARROW_LIBRARIES} COMMAND mkdir -p ${ARROW_INCLUDE_DIR} COMMAND cp -r ${ARROW_SOURCES_DIR}/src/extern_arrow/cpp/src/arrow ${ARROW_INCLUDE_DIR}/
40+
INSTALL_COMMAND
41+
mkdir -p ${ARROW_INSTALL_DIR}/lib/
42+
COMMAND cp ${ARROW_SOURCES_DIR}/src/extern_arrow/cpp/build/release/libarrow.a ${ARROW_LIBRARIES}
43+
COMMAND cp ${ARROW_SOURCES_DIR}/src/extern_arrow/cpp/build/release/libparquet.a ${ARROW_INSTALL_DIR}/lib/
44+
COMMAND cp ${ARROW_SOURCES_DIR}/src/extern_arrow/cpp/build/release/libarrow_acero.a ${ARROW_INSTALL_DIR}/lib/
45+
COMMAND cp ${ARROW_SOURCES_DIR}/src/extern_arrow/cpp/build/release/libarrow_bundled_dependencies.a ${ARROW_INSTALL_DIR}/lib/
46+
COMMAND mkdir -p ${ARROW_INCLUDE_DIR}
47+
COMMAND cp -r ${ARROW_SOURCES_DIR}/src/extern_arrow/cpp/src/arrow ${ARROW_INCLUDE_DIR}/
48+
COMMAND cp -r ${ARROW_SOURCES_DIR}/src/extern_arrow/cpp/src/parquet ${ARROW_INCLUDE_DIR}/
3849
)
3950

51+
ADD_DEPENDENCIES(extern_arrow zlib snappy zstd lz4 re2 protobuf rapidjson)
4052
ADD_LIBRARY(arrow STATIC IMPORTED GLOBAL)
4153
SET_PROPERTY(TARGET arrow PROPERTY IMPORTED_LOCATION ${ARROW_LIBRARIES})
42-
ADD_DEPENDENCIES(arrow extern_arrow)
54+
ADD_LIBRARY(parquet STATIC IMPORTED GLOBAL)
55+
SET_PROPERTY(TARGET parquet PROPERTY IMPORTED_LOCATION ${ARROW_PARQUET_LIB})
56+
ADD_LIBRARY(acero STATIC IMPORTED GLOBAL)
57+
SET_PROPERTY(TARGET acero PROPERTY IMPORTED_LOCATION ${ARROW_ACERO_LIB})
58+
ADD_LIBRARY(arrow_deps STATIC IMPORTED GLOBAL)
59+
SET_PROPERTY(TARGET arrow_deps PROPERTY IMPORTED_LOCATION ${ARROW_BUNDLED_DEP_LIB})
60+
ADD_DEPENDENCIES(arrow parquet acero arrow_deps extern_arrow)

‎cmake/boost.cmake

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ set_directory_properties(PROPERTIES CLEAN_NO_CUSTOM 1)
4141
include_directories(${BOOST_INCLUDE_DIR})
4242

4343
FILE(WRITE ${BOOST_DOWNLOAD_DIR}/build.sh
44-
"cd ${BOOST_DOWNLOAD_DIR} && mkdir -p build && sh bootstrap.sh --prefix=${BOOST_INSTALL_DIR} --with-libraries=thread,filesystem,regex,system && ./b2 install"
44+
"cd ${BOOST_DOWNLOAD_DIR} && mkdir -p build && sh bootstrap.sh --prefix=${BOOST_INSTALL_DIR} --with-libraries=thread,filesystem,regex,system && ./b2 install --quiet"
4545
)
4646

4747
ExternalProject_Add(

‎cmake/rocksdb.cmake

+1
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ ExternalProject_Add(
6767
CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${ROCKSDB_INSTALL_DIR}
6868
-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
6969
-DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
70+
BUILD_COMMAND make -j2
7071
)
7172

7273
ADD_DEPENDENCIES(extern_rocksdb zlib snappy zstd lz4 gflags liburing)

‎include/common/baikal_heartbeat.h

+10-3
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,16 @@ struct SubTableIds {
4040

4141
class BaikalHeartBeat {
4242
public:
43-
static void construct_heart_beat_request(pb::BaikalHeartBeatRequest& request, bool is_backup = false);
44-
static void process_heart_beat_response_sync(const pb::BaikalHeartBeatResponse& response);
45-
static void process_heart_beat_response(const pb::BaikalHeartBeatResponse& response, bool is_backup = false);
43+
// 主Meta使用
44+
static void construct_heart_beat_request(
45+
pb::BaikalHeartBeatRequest& request, bool is_backup = false);
46+
// 非主Meta使用
47+
static void construct_heart_beat_request(
48+
std::unordered_map<int64_t, pb::BaikalHeartBeatRequest>& meta_request_map);
49+
static void process_heart_beat_response_sync(
50+
const pb::BaikalHeartBeatResponse& response, const int64_t meta_id = 0);
51+
static void process_heart_beat_response(
52+
const pb::BaikalHeartBeatResponse& response, bool is_backup = false, const int64_t meta_id = 0);
4653
};
4754

4855
class BaseBaikalHeartBeat {

0 commit comments

Comments
 (0)