for xfs clone poc code

WORKSPACE

@@ -20,7 +20,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 
 git_repository(
     name = "com_github_baidu_braft",
-    remote = "https://github.com/baidu/braft",
+    remote = "https://ghproxy.com/https://github.com/baidu/braft",
     commit = "e255c0e4b18d1a8a5d484d4b647f41ff1385ef1e",
 )
 
@@ -36,7 +36,7 @@ http_archive(
     name = "com_google_protobuf",
     sha256 = "cef7f1b5a7c5fba672bec2a319246e8feba471f04dcebfe362d55930ee7c1c30",
     strip_prefix = "protobuf-3.5.0",
-    urls = ["https://github.com/google/protobuf/archive/v3.5.0.zip"],
+    urls = ["https://ghproxy.com/https://github.com/google/protobuf/archive/v3.5.0.zip"],
 )
 
 bind(
@@ -48,7 +48,7 @@ bind(
 new_git_repository(
     name = "com_google_googletest",
     build_file = "bazel/gmock.BUILD",
-    remote = "https://github.com/google/googletest",
+    remote = "https://ghproxy.com/https://github.com/google/googletest",
     tag = "release-1.8.0",
 )
 
@@ -61,7 +61,7 @@ bind(
 # brpc内BUILD文件在依赖glog时, 直接指定的依赖是"@com_github_google_glog//:glog"
 git_repository(
     name = "com_github_google_glog",
-    remote = "https://github.com/google/glog",
+    remote = "https://ghproxy.com/https://github.com/google/glog",
     commit = "4cc89c9e2b452db579397887c37f302fb28f6ca1",
     patch_args = ["-p1"],
     patches = ["//:thirdparties/glog/glog.patch"],
@@ -91,7 +91,7 @@ new_http_archive(
     name = "com_github_google_leveldb",
     build_file = "bazel/leveldb.BUILD",
     strip_prefix = "leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6",
-    url = "https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz",
+    url = "https://ghproxy.com/https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz",
 )
 
 bind(
@@ -101,7 +101,7 @@ bind(
 
 git_repository(
     name = "com_github_apache_brpc",
-    remote = "https://github.com/apache/incubator-brpc",
+    remote = "https://ghproxy.com/https://github.com/apache/incubator-brpc",
     commit = "1b9e00641cbec1c8803da6a1f7f555398c954cb0",
     patches = ["//:thirdparties/brpc/brpc.patch"],
     patch_args = ["-p1"],
@@ -131,7 +131,7 @@ bind(
 new_git_repository(
     name = "jsoncpp",
     build_file = "bazel/jsoncpp.BUILD",
-    remote = "https://github.com/open-source-parsers/jsoncpp.git",
+    remote = "https://ghproxy.com/https://github.com/open-source-parsers/jsoncpp.git",
     tag = "1.8.4",
 )
 

build.sh

@@ -18,16 +18,16 @@
 
 dir=`pwd`
 #step1 清除生成的目录和文件
-bazel clean
+#bazel clean
 rm -rf curvefs_python/BUILD
 rm -rf curvefs_python/tmplib/
 
-git submodule update --init
-if [ $? -ne 0 ]
-then
-    echo "submodule init failed"
-    exit
-fi
+#git submodule update --init
+#if [ $? -ne 0 ]
+#then
+#    echo "submodule init failed"
+#    exit
+#fi
 
 #step2 获取tag版本和git提交版本信息
 #获取tag版本
@@ -90,24 +90,25 @@ fi
 echo "gcc version : "`gcc -dumpversion`
 
 echo "start compile"
-cd ${dir}/thirdparties/etcdclient
-make clean
-make all
-if [ $? -ne 0 ]
-then
-    echo "make etcd client failed"
-    exit
-fi
-cd ${dir}
-
-cp ${dir}/thirdparties/etcdclient/libetcdclient.h ${dir}/include/etcdclient/etcdclient.h
+#cd ${dir}/thirdparties/etcdclient
+#make clean
+#make all
+#if [ $? -ne 0 ]
+#then
+#    echo "make etcd client failed"
+#    exit
+#fi
+#cd ${dir}
+#
+#cp ${dir}/thirdparties/etcdclient/libetcdclient.h ${dir}/include/etcdclient/etcdclient.h
 
 if [ `gcc -dumpversion | awk -F'.' '{print $1}'` -le 6 ]
 then
     bazelflags=''
 else
     bazelflags='--copt -faligned-new'
 fi
+echo "bazelflags=$bazelflags"
 
 if [ "$1" = "debug" ]
 then
@@ -138,31 +139,23 @@ then
     exit
 fi
 else
-bazel build ... --copt -DHAVE_ZLIB=1 --copt -O2 -s --define=with_glog=true \
+#bazel build ... --copt -DHAVE_ZLIB=1 --copt -O2 -s --define=with_glog=true \
+#bazel build //src/chunkserver:chunkserver --copt -DHAVE_ZLIB=1 --copt -O2 -s --define=with_glog=true \
+#bazel build //src/client:all --copt -DHAVE_ZLIB=1 --copt -O2 -s --define=with_glog=true \
+#bazel build //:curvebs-sdk --copt -DHAVE_ZLIB=1 --copt -O2 -s --define=with_glog=true \
+#bazel build --spawn_strategy=local //src/chunkserver:chunkserver --copt -DHAVE_ZLIB=1 --copt -O2 -s --define=with_glog=true \
+#cmd="bazel build  //src/tools:curve_format --copt -DHAVE_ZLIB=1 --copt -O2 -s --define=with_glog=true \
+cmd="bazel build  //src/chunkserver:chunkserver --copt -DHAVE_ZLIB=1 --copt -O2 -s --define=with_glog=true \
 --define=libunwind=true --copt -DGFLAGS_NS=google --copt \
 -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} \
---linkopt -L/usr/local/lib ${bazelflags}
+--linkopt -L/usr/local/lib ${bazelflags}"
+
+echo $cmd
+$cmd
 if [ $? -ne 0 ]
 then
     echo "build phase1 failed"
     exit
 fi
-bash ./curvefs_python/configure.sh
-if [ $? -ne 0 ]
-then
-    echo "configure failed"
-    exit
-fi
-bazel build curvefs_python:curvefs  --copt -DHAVE_ZLIB=1 --copt -O2 -s \
---define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google \
---copt \
--Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --linkopt \
--L${dir}/curvefs_python/tmplib/ --copt -DCURVEVERSION=${curve_version} \
---linkopt -L/usr/local/lib ${bazelflags}
-if [ $? -ne 0 ]
-then
-    echo "build phase2 failed"
-    exit
-fi
 fi
-echo "end compile"
+echo "end compile"

curve.code-workspace

@@ -0,0 +1,82 @@
+{
+	"folders": [
+		{
+			"path": "."
+		}
+	],
+	"settings": {
+		"files.associations": {
+			"*.rst": "rust",
+			"atomic": "cpp",
+			"deque": "cpp",
+			"string": "cpp",
+			"vector": "cpp",
+			"array": "cpp",
+			"*.tcc": "cpp",
+			"memory": "cpp",
+			"future": "cpp",
+			"istream": "cpp",
+			"ranges": "cpp",
+			"functional": "cpp",
+			"tuple": "cpp",
+			"utility": "cpp",
+			"list": "cpp",
+			"unordered_map": "cpp",
+			"unordered_set": "cpp",
+			"hash_map": "cpp",
+			"hash_set": "cpp",
+			"bitset": "cpp",
+			"cctype": "cpp",
+			"chrono": "cpp",
+			"cinttypes": "cpp",
+			"clocale": "cpp",
+			"cmath": "cpp",
+			"condition_variable": "cpp",
+			"csignal": "cpp",
+			"cstdarg": "cpp",
+			"cstddef": "cpp",
+			"cstdint": "cpp",
+			"cstdio": "cpp",
+			"cstdlib": "cpp",
+			"cstring": "cpp",
+			"ctime": "cpp",
+			"cwchar": "cpp",
+			"cwctype": "cpp",
+			"exception": "cpp",
+			"algorithm": "cpp",
+			"iterator": "cpp",
+			"map": "cpp",
+			"memory_resource": "cpp",
+			"numeric": "cpp",
+			"random": "cpp",
+			"ratio": "cpp",
+			"regex": "cpp",
+			"set": "cpp",
+			"system_error": "cpp",
+			"type_traits": "cpp",
+			"slist": "cpp",
+			"fstream": "cpp",
+			"initializer_list": "cpp",
+			"iomanip": "cpp",
+			"iosfwd": "cpp",
+			"iostream": "cpp",
+			"limits": "cpp",
+			"mutex": "cpp",
+			"new": "cpp",
+			"ostream": "cpp",
+			"sstream": "cpp",
+			"stdexcept": "cpp",
+			"streambuf": "cpp",
+			"thread": "cpp",
+			"cfenv": "cpp",
+			"typeinfo": "cpp",
+			"*.inc": "cpp",
+			"cerrno": "cpp",
+			"string_view": "cpp",
+			"valarray": "cpp",
+			"forward_list": "cpp",
+			"any": "cpp",
+			"optional": "cpp"
+		}
+	}
+}

src/chunkserver/chunkserver.cpp

@@ -37,6 +37,7 @@
 #include "src/chunkserver/braft_cli_service.h"
 #include "src/chunkserver/braft_cli_service2.h"
 #include "src/chunkserver/chunkserver_helper.h"
+#include "src/common/concurrent/task_thread_pool.h"
 #include "src/chunkserver/uri_paser.h"
 #include "src/chunkserver/raftsnapshot/curve_snapshot_attachment.h"
 #include "src/chunkserver/raftsnapshot/curve_file_service.h"
@@ -104,6 +105,7 @@ int ChunkServer::Run(int argc, char** argv) {
     curve::common::ExposeCurveVersion();
 
     // ============================初始化各模块==========================//
+    LOG(INFO) << "zyb xfs mod ChunkServer";
     LOG(INFO) << "Initializing ChunkServer modules";
 
     LOG_IF(FATAL, !conf.GetUInt32Value("global.min_io_alignment",
@@ -128,7 +130,7 @@ int ChunkServer::Run(int argc, char** argv) {
 
     // 初始化本地文件系统
     std::shared_ptr<LocalFileSystem> fs(
-        LocalFsFactory::CreateFs(FileSystemType::EXT4, ""));
+        LocalFsFactory::CreateFs(FileSystemType::XFS, ""));
     LocalFileSystemOption lfsOption;
     LOG_IF(FATAL, !conf.GetBoolValue(
         "fs.enable_renameat2", &lfsOption.enableRenameat2));
@@ -547,15 +549,21 @@ void ChunkServer::InitCopysetNodeOptions(
         &copysetNodeOptions->finishLoadMargin));
     LOG_IF(FATAL, !conf->GetUInt32Value("copyset.check_loadmargin_interval_ms",
         &copysetNodeOptions->checkLoadMarginIntervalMs));
+    LOG_IF(FATAL, !conf->GetUInt32Value("copyset.sync_concurrency",
+            &copysetNodeOptions->syncConcurrency));
 
     LOG_IF(FATAL, !conf->GetBoolValue(
         "copyset.enable_odsync_when_open_chunkfile",
         &copysetNodeOptions->enableOdsyncWhenOpenChunkFile));
-    if (!copysetNodeOptions->enableOdsyncWhenOpenChunkFile) {
-        LOG_IF(FATAL, !conf->GetUInt32Value("copyset.synctimer_interval_ms",
-            &copysetNodeOptions->syncTimerIntervalMs));
+    if (copysetNodeOptions->enableOdsyncWhenOpenChunkFile) {
+        LOG_IF(FATAL, !conf->GetUInt64Value("copyset.sync_chunk_limits",
+            &copysetNodeOptions->syncChunkLimit));
+        LOG_IF(FATAL, !conf->GetUInt64Value("copyset.sync_threshold",
+            &copysetNodeOptions->syncThreshold));
         LOG_IF(FATAL, !conf->GetUInt32Value("copyset.check_syncing_interval_ms",
             &copysetNodeOptions->checkSyncingIntervalMs));
+        LOG_IF(FATAL, !conf->GetUInt32Value("copyset.sync_trigger_seconds",
+                &copysetNodeOptions->syncTriggerSeconds));
     }
 }
 

src/chunkserver/config_info.h

@@ -111,6 +111,10 @@ struct CopysetNodeOptions {
 
     // 限制chunkserver启动时copyset并发恢复加载的数量,为0表示不限制
     uint32_t loadConcurrency = 0;
+    // chunkserver sync_thread_pool number of threads.
+    uint32_t syncConcurrency = 20;
+    // copyset trigger sync timeout
+    uint32_t syncTriggerSeconds = 25;
     // 检查copyset是否加载完成出现异常时的最大重试次数
     // 可能的异常：1.当前大多数副本还没起来；2.网络问题等导致无法获取leader
     // 3.其他的原因导致无法获取到leader的committed index
@@ -123,8 +127,10 @@ struct CopysetNodeOptions {
 
     // enable O_DSYNC when open chunkfile
     bool enableOdsyncWhenOpenChunkFile = false;
-    // sync timer timeout interval
-    uint32_t syncTimerIntervalMs = 30000u;
+    // syncChunkLimit default limit
+    uint64_t syncChunkLimit = 2 * 1024 * 1024;
+    // syncHighChunkLimit default limit = 64k
+    uint64_t syncThreshold = 64 * 1024;
     // check syncing interval
     uint32_t checkSyncingIntervalMs = 500u;