From 5ff22dda57b73bf266d7960a2f57e444793b7fc5 Mon Sep 17 00:00:00 2001 From: cnswee Date: Mon, 14 Aug 2023 16:04:32 +0800 Subject: [PATCH] Add QAT support in zfile Signed-off-by: cnswee --- .gitmodules | 3 + src/overlaybd/zfile/CMakeLists.txt | 9 +- src/overlaybd/zfile/compressor.cpp | 227 +++++++++++++----- src/overlaybd/zfile/lz4/lz4-qat.c | 46 ---- src/overlaybd/zfile/lz4/lz4-qat.h | 110 --------- src/overlaybd/zfile/thirdparty/CMakeLists.txt | 13 + 6 files changed, 191 insertions(+), 217 deletions(-) delete mode 100644 src/overlaybd/zfile/lz4/lz4-qat.c delete mode 100644 src/overlaybd/zfile/lz4/lz4-qat.h diff --git a/.gitmodules b/.gitmodules index cc9c9aab..ae97e556 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,3 +8,6 @@ [submodule "src/overlaybd/zfile/thirdparty/isa-l"] path = src/overlaybd/zfile/thirdparty/isa-l url = https://github.com/intel/isa-l.git +[submodule "src/overlaybd/zfile/thirdparty/QATzip"] + path = src/overlaybd/zfile/thirdparty/QATzip + url = https://github.com/intel/QATzip.git diff --git a/src/overlaybd/zfile/CMakeLists.txt b/src/overlaybd/zfile/CMakeLists.txt index 6d7eeb41..729bd74d 100644 --- a/src/overlaybd/zfile/CMakeLists.txt +++ b/src/overlaybd/zfile/CMakeLists.txt @@ -35,17 +35,22 @@ if(ENABLE_DSA OR ENABLE_ISAL) target_link_libraries(crc32_lib -lisal) endif() endif() -set (CMAKE_CXX_STANDARD 14) add_library(zfile_lib STATIC ${SOURCE_ZFILE} ${SOURCE_LZ4}) target_link_libraries(zfile_lib photon_static crc32_lib ${LIBZSTD}) if (ENABLE_QAT) + add_subdirectory(thirdparty) + add_dependencies(zfile_lib thirdparty_lib) + # target_link_directories(zfile_lib PUBLIC ${LIBRARY_OUTPUT_PATH}) + # target_include_directories(zfile_lib PUBLIC ${LIBRARY_OUTPUT_PATH}/include) target_compile_definitions(zfile_lib PUBLIC -DENABLE_QAT) - target_link_libraries(zfile_lib -lpthread -lpci) + target_link_libraries(zfile_lib -lpci -lqatzip) #target_link_libraries(zfile_lib -lqat_s -lusdm_drv_s -lpthread -lpci) endif() +set (CMAKE_CXX_STANDARD 14) + if (BUILD_TESTING) add_subdirectory(test) endif () diff --git a/src/overlaybd/zfile/compressor.cpp b/src/overlaybd/zfile/compressor.cpp index 85041d38..486ff431 100644 --- a/src/overlaybd/zfile/compressor.cpp +++ b/src/overlaybd/zfile/compressor.cpp @@ -25,7 +25,10 @@ #include "photon/fs/filesystem.h" #ifdef ENABLE_QAT -#include "lz4/lz4-qat.h" +#include +#include +#include +#include extern "C" { #include } @@ -46,7 +49,7 @@ class BaseCompressor : public ICompressor { vector compressed_data; vector uncompressed_data; - const int DEFAULT_N_BATCH = 256; + const int DEFAULT_N_BATCH = 1; virtual int init(const CompressArgs *args) { auto opt = &args->opt; @@ -129,20 +132,24 @@ class BaseCompressor : public ICompressor { } }; -class LZ4Compressor : public BaseCompressor { -public: - bool qat_enable = false; #ifdef ENABLE_QAT - LZ4_qat_param *pQat = nullptr; +struct QzSessionDeleter { + void operator()(struct QzSession_S *session); +}; + +void QzSessionDeleter::operator()(struct QzSession_S *session) { + qzTeardownSession(session); + delete session; +} #endif - ~LZ4Compressor() { +class LZ4Compressor : public BaseCompressor { +public: #ifdef ENABLE_QAT - if (pQat) { - qat_uninit(pQat); - delete pQat; - } + using session_ptr = std::unique_ptr; #endif + bool qat_enable = false; + ~LZ4Compressor() { } bool check_qat() { @@ -179,8 +186,6 @@ class LZ4Compressor : public BaseCompressor { max_dst_size = LZ4_compressBound(src_blk_size); #ifdef ENABLE_QAT if (check_qat()) { - pQat = new LZ4_qat_param(); - qat_init(pQat); qat_enable = true; } #endif @@ -192,69 +197,173 @@ class LZ4Compressor : public BaseCompressor { return (qat_enable ? DEFAULT_N_BATCH : 1); } - virtual int do_compress(size_t *src_chunk_len, size_t *dst_chunk_len, - size_t dst_buffer_capacity, size_t nblock) override { + int do_compress(size_t *src_chunk_len, size_t *dst_chunk_len, size_t dst_buffer_capacity, + size_t nblock) override; - int ret = 0; + int do_decompress(size_t *src_chunk_len, size_t *dst_chunk_len, size_t dst_buffer_capacity, + size_t n) override; #ifdef ENABLE_QAT - if (qat_enable) { - ret = LZ4_compress_qat(pQat, &raw_data[0], src_chunk_len, &compressed_data[0], - dst_chunk_len, n); - if (ret < 0) { - LOG_ERROR_RETURN(EFAULT, -1, "LZ4 compress data failed. (retcode: `).", ret); - } - return ret; +private: + session_ptr get_session(); + friend struct cached_session_t; + std::vector sessions; + std::mutex mutex; +#endif +}; + +#ifdef ENABLE_QAT +static bool setup_session(LZ4Compressor::session_ptr &session) { + int rc; + rc = qzInit(session.get(), QZ_SW_BACKUP_DEFAULT); + if (rc != QZ_OK && rc != QZ_DUPLICATE) + return false; + QzSessionParamsLZ4_T params; + rc = qzGetDefaultsLZ4(¶ms); + if (rc != QZ_OK) + return false; + // params.data_fmt = QZ_LZ4; + params.common_params.comp_algorithm = QZ_LZ4; + params.common_params.comp_lvl = 1; + params.common_params.direction = QZ_DIR_BOTH; + rc = qzSetupSessionLZ4(session.get(), ¶ms); + if (rc != QZ_OK) + return false; + + return true; +} + +// put the session back to the session pool in a RAII manner +struct cached_session_t { + cached_session_t(const cached_session_t &); + + cached_session_t(LZ4Compressor *accel, LZ4Compressor::session_ptr &&sess) + : accel{accel}, session{std::move(sess)} { + } + + ~cached_session_t() { + std::lock_guard lock{accel->mutex}; + // if the cache size is still under its upper bound, the current session is put into + // accel->sessions. otherwise it's released right + uint64_t sessions_num = 256; + // g_ceph_context->_conf.get_val("qat_compressor_session_max_number"); + if (accel->sessions.size() < sessions_num) { + accel->sessions.push_back(std::move(session)); } + } + + struct QzSession_S *get() { + assert(static_cast(session)); + return session.get(); + } + + LZ4Compressor *accel; + LZ4Compressor::session_ptr session; +}; + +LZ4Compressor::session_ptr LZ4Compressor::get_session() { + { + std::lock_guard lock{mutex}; + if (!sessions.empty()) { + auto session = std::move(sessions.back()); + sessions.pop_back(); + return session; + } + } + + // If there are no available session to use, we try allocate a new + // session. + session_ptr session(new struct QzSession_S()); + memset(session.get(), 0, sizeof(struct QzSession_S)); + if (setup_session(session)) { + return session; + } else { + return nullptr; + } +} #endif - for (size_t i = 0; i < nblock; i++) { - ret = - LZ4_compress_default((const char *)uncompressed_data[i], (char *)compressed_data[i], - src_chunk_len[i], dst_buffer_capacity / nblock); - dst_chunk_len[i] = ret; - if (ret < 0) { - LOG_ERROR_RETURN(EFAULT, -1, "LZ4 compress data failed. (retcode: `).", ret); - } - if (ret == 0) { - LOG_ERROR_RETURN( - EFAULT, -1, - "Compression worked, but was stopped because the *dst couldn't hold all the information."); +int LZ4Compressor::do_compress(size_t *src_chunk_len, size_t *dst_chunk_len, + size_t dst_buffer_capacity, size_t nblock) { + + int ret = 0; +#ifdef ENABLE_QAT + if (qat_enable) { + auto s = get_session(); // get a session from the pool + if (!s) { + LOG_INFO("get_session failed!"); + return -1; // session initialization failed + } + auto session = + cached_session_t{this, std::move(s)}; // returns to the session pool on destruction + + for (size_t i = 0; i < nblock; i++) { + unsigned int len = src_chunk_len[i]; + unsigned int out_len = qzMaxCompressedLength(len, session.get()); + int rc = qzCompress(session.get(), uncompressed_data[i], (&len), compressed_data[i], + (&out_len), 1); + dst_chunk_len[i] = out_len; + if (rc != QZ_OK) { + LOG_INFO("qzCompress failed!, rc: `", rc); + return -1; } } return 0; } +#endif + for (size_t i = 0; i < nblock; i++) { + ret = LZ4_compress_default((const char *)uncompressed_data[i], (char *)compressed_data[i], + src_chunk_len[i], dst_buffer_capacity / nblock); - int do_decompress(size_t *src_chunk_len, size_t *dst_chunk_len, size_t dst_buffer_capacity, - size_t n) override { + dst_chunk_len[i] = ret; + if (ret < 0) { + LOG_ERROR_RETURN(EFAULT, -1, "LZ4 compress data failed. (retcode: `).", ret); + } + if (ret == 0) { + LOG_ERROR_RETURN( + EFAULT, -1, + "Compression worked, but was stopped because the *dst couldn't hold all the information."); + } + } + return 0; +} +int LZ4Compressor::do_decompress(size_t *src_chunk_len, size_t *dst_chunk_len, + size_t dst_buffer_capacity, size_t n) { - int ret = 0; + int ret = 0; #ifdef ENABLE_QAT - if (qat_enable) { - ret = LZ4_decompress_qat(pQat, &compressed_data[0], src_chunk_len, - &uncompressed_data[0], dst_chunk_len, n); - if (ret < 0) { - LOG_ERROR_RETURN(EFAULT, -1, "LZ4 decompress data failed. (retcode: `).", ret); - } - return ret; + if (qat_enable) { + auto s = get_session(); // get a session from the pool + if (!s) { + return -1; // session initialization failed } -#endif - for (size_t i = 0; i < n; i++) { - ret = - LZ4_decompress_safe((const char *)compressed_data[i], (char *)uncompressed_data[i], - src_chunk_len[i], dst_buffer_capacity / n); + auto session = + cached_session_t{this, std::move(s)}; // returns to the session pool on destruction - dst_chunk_len[i] = ret; - if (ret < 0) { - LOG_ERROR_RETURN(EFAULT, -1, "LZ4 decompress data failed. (retcode: `).", ret); - } - if (ret == 0) { - LOG_ERROR_RETURN(EFAULT, -1, - "LZ4 decompress returns 0. THIS SHOULD BE NEVER HAPPEN!"); - } + for (size_t i = 0; i < n; i++) { + int rc = qzDecompress(session.get(), compressed_data[i], + reinterpret_cast(&src_chunk_len[i]), + uncompressed_data[i], + reinterpret_cast(&dst_chunk_len[i])); + if (rc != QZ_OK) + return -1; } return 0; } -}; +#endif + for (size_t i = 0; i < n; i++) { + ret = LZ4_decompress_safe((const char *)compressed_data[i], (char *)uncompressed_data[i], + src_chunk_len[i], dst_buffer_capacity / n); + + dst_chunk_len[i] = ret; + if (ret < 0) { + LOG_ERROR_RETURN(EFAULT, -1, "LZ4 decompress data failed. (retcode: `).", ret); + } + if (ret == 0) { + LOG_ERROR_RETURN(EFAULT, -1, "LZ4 decompress returns 0. THIS SHOULD BE NEVER HAPPEN!"); + } + } + return 0; +} class Compressor_zstd : public BaseCompressor { public: diff --git a/src/overlaybd/zfile/lz4/lz4-qat.c b/src/overlaybd/zfile/lz4/lz4-qat.c deleted file mode 100644 index b8b33466..00000000 --- a/src/overlaybd/zfile/lz4/lz4-qat.c +++ /dev/null @@ -1,46 +0,0 @@ -#include "lz4.h" -#include "lz4-qat.h" -#include - -#define sleeptime 100 - -int gDebugParam = 1; - - -int qat_init(LZ4_qat_param *pQat) { - int32_t status = 0; - - return (int)status; -} - -int qat_uninit(LZ4_qat_param *pQat) { - int32_t status = 0; - - return status; -} - -// compression operation. - -LZ4LIB_API int LZ4_compress_qat(LZ4_qat_param *pQat, const unsigned char *const raw_data[], - size_t src_chunk_len[], unsigned char *compressed_data[], - size_t dst_chunk_len[], size_t n) { - int32_t status = 0; - for (size_t i = 0; i < n; i++) { - int ret = LZ4_compress_default((const char *)raw_data[i], (char *)compressed_data[i], - src_chunk_len[i], 4096); - dst_chunk_len[i] = ret; - } - return status; -} - -LZ4LIB_API int LZ4_decompress_qat(LZ4_qat_param *pQat, const unsigned char *const raw_data[], - size_t src_chunk_len[], unsigned char *decompressed_data[], - size_t dst_chunk_len[], size_t n){ - int32_t status = 0; - for (size_t i = 0; i < n; i++) { - int ret = LZ4_decompress_safe((const char *)raw_data[i], (char *)decompressed_data[i], - src_chunk_len[i], 4096); - dst_chunk_len[i] = ret; - } - return status; -} diff --git a/src/overlaybd/zfile/lz4/lz4-qat.h b/src/overlaybd/zfile/lz4/lz4-qat.h deleted file mode 100644 index 5c539cdf..00000000 --- a/src/overlaybd/zfile/lz4/lz4-qat.h +++ /dev/null @@ -1,110 +0,0 @@ -/* - * LZ4 - Fast LZ compression algorithm - * Header File - * Copyright (C) 2011-present, Yann Collet. - - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - LZ4 homepage : http://www.lz4.org - - LZ4 source repository : https://github.com/lz4/lz4 -*/ -#if defined(__cplusplus) -extern "C" { -#endif - -#ifndef LZ4_QAT_H_2983827168210 -#define LZ4_QAT_H_2983827168210 - -#include -#include -/* --- Dependency --- */ - -#ifndef LZ4LIB_VISIBILITY -#if defined(__GNUC__) && (__GNUC__ >= 4) -#define LZ4LIB_VISIBILITY __attribute__((visibility("default"))) -#else -#define LZ4LIB_VISIBILITY -#endif -#endif -#if defined(LZ4_DLL_EXPORT) && (LZ4_DLL_EXPORT == 1) -#define LZ4LIB_API __declspec(dllexport) LZ4LIB_VISIBILITY -#elif defined(LZ4_DLL_IMPORT) && (LZ4_DLL_IMPORT == 1) -#define LZ4LIB_API \ - __declspec(dllimport) \ - LZ4LIB_VISIBILITY /* It isn't required but allows to generate better code, saving a \ - function pointer load from the IAT and an indirect jump.*/ -#else -#define LZ4LIB_API LZ4LIB_VISIBILITY -#endif - -typedef struct _LZ4_qat_param LZ4_qat_param; -struct _LZ4_qat_param { - -}; - -/*-************************************ - * Simple Functions - **************************************/ -/*! LZ4_compress_qat() : - Compresses 'srcSize' bytes from buffer 'src' - into already allocated 'dst' buffer of size 'dstCapacity'. - Compression is guaranteed to succeed if 'dstCapacity' >= LZ4_compressBound(srcSize). - It also runs faster, so it's a recommended setting. - If the function cannot compress 'src' into a more limited 'dst' budget, - compression stops *immediately*, and the function result is zero. - Note : as a consequence, 'dst' content is not valid. - Note 2 : This function is protected against buffer overflow scenarios (never writes outside - 'dst' buffer, nor read outside 'source' buffer). srcSize : max supported value is - LZ4_MAX_INPUT_SIZE. dstCapacity : size of buffer 'dst' (which must be already allocated) return - : the number of bytes written into buffer 'dst' (necessarily <= dstCapacity) - or 0 if compression fails */ -LZ4LIB_API int LZ4_compress_qat(LZ4_qat_param *pQat, const unsigned char *const raw_data[], - size_t src_chunk_len[], unsigned char *compressed_data[], - size_t dst_chunk_len[], size_t n); - -/*! LZ4_decompress_qat() : - compressedSize : is the exact complete size of the compressed block. - dstCapacity : is the size of destination buffer, which must be already allocated. - return : the number of bytes decompressed into destination buffer (necessarily <= dstCapacity) - If destination buffer is not large enough, decoding will stop and output an error code - (negative value). If the source stream is detected malformed, the function will stop decoding and - return a negative result. This function is protected against malicious data packets. -*/ -LZ4LIB_API int LZ4_decompress_qat(LZ4_qat_param *pQat, const unsigned char *const raw_data[], - size_t src_chunk_len[], unsigned char *decompressed_data[], - size_t dst_chunk_len[], size_t n); - - -int qat_init(LZ4_qat_param *pQat); - -int qat_uninit(LZ4_qat_param *pQat); - -#endif /* LZ4_QAT_H_2983827168210 */ - -#if defined(__cplusplus) -} -#endif \ No newline at end of file diff --git a/src/overlaybd/zfile/thirdparty/CMakeLists.txt b/src/overlaybd/zfile/thirdparty/CMakeLists.txt index 4dec6f2a..5ff66824 100644 --- a/src/overlaybd/zfile/thirdparty/CMakeLists.txt +++ b/src/overlaybd/zfile/thirdparty/CMakeLists.txt @@ -20,4 +20,17 @@ if(ENABLE_ISAL) COMMAND cp -r ${THIRDPARTY_PATH}/isa-l/.libs/libisal.a ${LIBRARY_OUTPUT_PATH}/ COMMAND mkdir -p ${LIBRARY_OUTPUT_PATH}/include && cp ${THIRDPARTY_PATH}/isa-l/include/crc.h ${LIBRARY_OUTPUT_PATH}/include/ ) +endif() + +if(ENABLE_QAT) + add_custom_command(TARGET thirdparty_lib + COMMAND cd ${THIRDPARTY_PATH}/QATzip && git checkout b908048e13e076f85b2cec2e72b366ae755c2fb7 + COMMAND cd ${THIRDPARTY_PATH}/QATzip && ./autogen.sh + COMMAND cd ${THIRDPARTY_PATH}/QATzip && ./configure --with-ICP_ROOT=$ENV{ICP_ROOT} + COMMAND cd ${THIRDPARTY_PATH}/QATzip && make clean + COMMAND cd ${THIRDPARTY_PATH}/QATzip && make + COMMAND cd ${THIRDPARTY_PATH}/QATzip && make install + COMMAND cp -r ${THIRDPARTY_PATH}/QATzip/src/.libs/libqatzip.a ${LIBRARY_OUTPUT_PATH}/ + COMMAND mkdir -p ${LIBRARY_OUTPUT_PATH}/include && cp ${THIRDPARTY_PATH}/QATzip/include/qatzip.h ${LIBRARY_OUTPUT_PATH}/include/ + ) endif() \ No newline at end of file