Started by timer [EnvInject] - Loading node environment variables. Building remotely on plasma_dev_test1 (plasmadevtest) in workspace /data/jenkins/workspace/workspace/centos-2i_plasma_API_tests No emails were triggered. [centos-2i_plasma_API_tests] $ /bin/sh -xe /tmp/jenkins8977181746178887225.sh + set -e + ulimit -a core file size (blocks, -c) 0 data seg size (kbytes, -d) unlimited scheduling priority (-e) 0 file size (blocks, -f) unlimited pending signals (-i) 107692 max locked memory (kbytes, -l) 64 max memory size (kbytes, -m) unlimited open files (-n) 102400 pipe size (512 bytes, -p) 8 POSIX message queues (bytes, -q) 819200 real-time priority (-r) 0 stack size (kbytes, -s) 8192 cpu time (seconds, -t) unlimited max user processes (-u) 107692 virtual memory (kbytes, -v) unlimited file locks (-x) unlimited + [[ None == \N\o\n\e ]] + curl -s 'http://dbapi.build.couchbase.com:8000/v1/products/couchbase-server/releases/cheshire-cat/versions/7.0.0/builds?filter=last_unit_sanity' ++ xargs +++ jq .build_num build.json ++ echo '"5306"' + build_num=5306 + version_number=7.0.0-5306 + echo 7.0.0-5306 7.0.0-5306 ++ echo 7.0.0-5306 ++ cut -d - -f1 ++ cut -d . -f1,2 + code=7.0 + '[' 7.0 == 4.1 ']' + '[' 7.0 == 4.5 ']' + '[' 7.0 == 5.0 ']' + '[' 7.0 == 5.1 ']' + '[' 7.0 == 5.5 ']' + '[' 7.0 == 6.0 ']' + '[' 7.0 == 6.5 ']' + '[' 7.0 == 7.0 ']' + branch=cheshire-cat + base_url=http://172.23.126.166/builds/latestbuilds/couchbase-server ++ echo 7.0.0-5306 ++ cut -d - -f2 + build=5306 + mkdir -p /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat + cd /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat + manifest=couchbase-server-7.0.0-5306-manifest.xml + wget http://172.23.126.166/builds/latestbuilds/couchbase-server/cheshire-cat/5306/couchbase-server-7.0.0-5306-manifest.xml --2021-09-13 16:00:00-- http://172.23.126.166/builds/latestbuilds/couchbase-server/cheshire-cat/5306/couchbase-server-7.0.0-5306-manifest.xml Connecting to 172.23.126.166:80... connected. HTTP request sent, awaiting response... 200 OK Length: 20444 (20K) [text/xml] Saving to: ‘couchbase-server-7.0.0-5306-manifest.xml.96’ 0K .......... ......... 100% 108M=0s 2021-09-13 16:00:01 (108 MB/s) - ‘couchbase-server-7.0.0-5306-manifest.xml.96’ saved [20444/20444] + mkdir -p /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/.repo + /root/bin/repo init -u git://github.com/couchbase/manifest.git -g all,enterprise -m couchbase-server-7.0.0-5306-manifest.xml Updating 4b5e1d4..4252a86 Fast-forward ... A new version of repo (2.15) is available. ... You should upgrade soon: cp /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/.repo/repo/repo /root/bin/repo repo: warning: git-1.9.1+ will soon be required; please upgrade your version of git to maintain support. project .repo/manifests/ repo has been initialized in /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat + cd /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat + cp couchbase-server-7.0.0-5306-manifest.xml .repo/manifests/ + rm -rf goproj/src/github.com/couchbase/plasma + /root/bin/repo sync -j6 --force-sync goproj/src/github.com/couchbase/nitro ... A new version of repo (2.15) is available. ... You should upgrade soon: cp /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/.repo/repo/repo /root/bin/repo error: refs is different in /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/nitro/.git vs /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/.repo/projects/goproj/src/github.com/couchbase/nitro.git repo sync has finished successfully. + cd /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat + export GOPATH=/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/ + GOPATH=/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/ + export GOPATH=/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj//src/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/godeps/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/godeps/src:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src + GOPATH=/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj//src/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/godeps/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/godeps/src:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src [centos-2i_plasma_API_tests] $ /bin/sh -xe /tmp/jenkins8053822935038225770.sh + echo '---------------------------- TESTS Plasma API Longevity Tests -----------------------' ---------------------------- TESTS Plasma API Longevity Tests ----------------------- + [[ None == \N\o\n\e ]] + curl -s 'http://dbapi.build.couchbase.com:8000/v1/products/couchbase-server/releases/cheshire-cat/versions/7.0.0/builds?filter=last_unit_sanity' ++ xargs +++ jq .build_num build.json ++ echo '"5306"' + build_num=5306 + version_number=7.0.0-5306 + echo 7.0.0-5306 7.0.0-5306 ++ echo 7.0.0-5306 ++ cut -d - -f1 ++ cut -d . -f1,2 + code=7.0 + '[' 7.0 == 4.1 ']' + '[' 7.0 == 4.5 ']' + '[' 7.0 == 5.0 ']' + '[' 7.0 == 5.1 ']' + '[' 7.0 == 5.5 ']' + '[' 7.0 == 6.0 ']' + '[' 7.0 == 6.5 ']' + '[' 7.0 == 7.0 ']' + branch=cheshire-cat + cbconfig_path=/data/jenkins/workspace/centos-2i_plasma_API_tests/config/cen_func_2i_conf.json + workspace=/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat + mkdir -p /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/go + export GOROOT=/usr/local/go + GOROOT=/usr/local/go + export PATH=/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/usr/sbin + PATH=/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/usr/sbin + export GOPATH=/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/godeps/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/go/ + GOPATH=/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/godeps/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/go/ + export GOPATH=/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/godeps/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/go/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/godeps/src:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/go/src + GOPATH=/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/godeps/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/go/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/godeps/src:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/go/src + export GO111MODULE=off + GO111MODULE=off + export C_INCLUDE_PATH=/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/platform/include/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/sigar/include:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/forestdb/include:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/build/tlm/deps:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/build/tlm/deps/curl.exploded/include:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/sigar/include:/opt/couchbase/include/ + C_INCLUDE_PATH=/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/platform/include/:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/sigar/include:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/forestdb/include:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/build/tlm/deps:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/build/tlm/deps/curl.exploded/include:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/sigar/include:/opt/couchbase/include/ + export 'CGO_LDFLAGS=-L /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/sigar/src -L /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/install/lib -ljemalloc' + CGO_LDFLAGS='-L /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/sigar/src -L /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/install/lib -ljemalloc' + export LD_LIBRARY_PATH=/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/sigar/src:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/install/lib + LD_LIBRARY_PATH=/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/sigar/src:/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/install/lib + export 'CGO_CFLAGS=-I/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/sigar/include -I/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/forestdb/include -I/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/tlm/deps/jemalloc.exploded/include -DJEMALLOC' + CGO_CFLAGS='-I/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/sigar/include -I/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/forestdb/include -I/data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/tlm/deps/jemalloc.exploded/include -DJEMALLOC' + go env -w GOPROXY=https://proxy.golang.org,direct + echo ---------------------------------------- ---------------------------------------- + go version go version go1.13 linux/amd64 + echo ---------------------------------------- ---------------------------------------- + cd /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat + pwd /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat + cd sigar/src + cmake . CMake Warning (dev) in CMakeLists.txt: No project() command is present. The top-level CMakeLists.txt file must contain a literal, direct call to the project() command. Add a line of code such as project(ProjectName) near the top of the file, but after cmake_minimum_required(). CMake is pretending there is a "project(Project)" command on the first line. This warning is for project developers. Use -Wno-dev to suppress it. CMake Warning (dev) in CMakeLists.txt: No cmake_minimum_required command is present. A line of code such as cmake_minimum_required(VERSION 3.20) should be added at the top of the file. The version specified may be lower if you wish to support older CMake versions for this project. For more information run "cmake --help-policy CMP0000". This warning is for project developers. Use -Wno-dev to suppress it. -- Configuring done -- Generating done -- Build files have been written to: /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/sigar/src + make Consolidate compiler generated dependencies of target sigar [100%] Built target sigar + cd /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/ + rm -rf plasma + rm -rf nitro + git clone https://e70e12cf2ec16ec0660e814d67238cbd226eb91d@github.com/couchbase/plasma.git Cloning into 'plasma'... + git clone https://e70e12cf2ec16ec0660e814d67238cbd226eb91d@github.com/couchbase/nitro.git Cloning into 'nitro'... + cd /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/nitro + git checkout unstable Switched to a new branch 'unstable' Branch unstable set up to track remote branch unstable from origin. + git fetch + git pull origin unstable From https://github.com/couchbase/nitro * branch unstable -> FETCH_HEAD Already up-to-date. + cd /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi + git checkout unstable Switched to a new branch 'unstable' Branch unstable set up to track remote branch unstable from origin. + git fetch + git pull origin unstable From https://github.com/couchbase/plasma * branch unstable -> FETCH_HEAD Already up-to-date. + rm -rf /data/jenkins/workspace/centos-2i_plasma_API_tests/reports + rm -rf /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/output.txt + rm -rf test_output.xml + mkdir -p /data/jenkins/workspace/centos-2i_plasma_API_tests/reports + [[ None == \N\o\n\e ]] ++ ls config/indexes ++ wc -l + num_config=13 + echo 13 13 ++ expr 1254 % '(' 13 - 1 ')' + 1 + config_json_index=7 + declare -a arrConfigJson + for file in 'config/indexes/*.json' + arrConfigJson=("${arrConfigJson[@]}" "$file") + for file in 'config/indexes/*.json' + arrConfigJson=("${arrConfigJson[@]}" "$file") + for file in 'config/indexes/*.json' + arrConfigJson=("${arrConfigJson[@]}" "$file") + for file in 'config/indexes/*.json' + arrConfigJson=("${arrConfigJson[@]}" "$file") + for file in 'config/indexes/*.json' + arrConfigJson=("${arrConfigJson[@]}" "$file") + for file in 'config/indexes/*.json' + arrConfigJson=("${arrConfigJson[@]}" "$file") + for file in 'config/indexes/*.json' + arrConfigJson=("${arrConfigJson[@]}" "$file") + for file in 'config/indexes/*.json' + arrConfigJson=("${arrConfigJson[@]}" "$file") + for file in 'config/indexes/*.json' + arrConfigJson=("${arrConfigJson[@]}" "$file") + for file in 'config/indexes/*.json' + arrConfigJson=("${arrConfigJson[@]}" "$file") + for file in 'config/indexes/*.json' + arrConfigJson=("${arrConfigJson[@]}" "$file") + for file in 'config/indexes/*.json' + arrConfigJson=("${arrConfigJson[@]}" "$file") + echo config/indexes/n100_basic16.json config/indexes/n100_basic1K.json config/indexes/n100_basic64.json config/indexes/n100_low5.json config/indexes/n10_basic16.json config/indexes/n10_basic1K.json config/indexes/n10_basic4K.json config/indexes/n10_basic64K.json config/indexes/n10_low5.json config/indexes/n10_mixed1.json config/indexes/n10_mixed2.json config/indexes/n10_multigen16.json config/indexes/n100_basic16.json config/indexes/n100_basic1K.json config/indexes/n100_basic64.json config/indexes/n100_low5.json config/indexes/n10_basic16.json config/indexes/n10_basic1K.json config/indexes/n10_basic4K.json config/indexes/n10_basic64K.json config/indexes/n10_low5.json config/indexes/n10_mixed1.json config/indexes/n10_mixed2.json config/indexes/n10_multigen16.json + config_json=config/indexes/n10_basic64K.json + echo config/indexes/n10_basic64K.json config/indexes/n10_basic64K.json + echo 'while :; do curl -sK -v http://localhost:8080/debug/pprof/heap > /data/jenkins/workspace/centos-2i_plasma_API_tests/heap`date +%Y%m%d%H%M%S`.out 2>&1; sleep 600; done' + nohup bash /data/jenkins/workspace/centos-2i_plasma_API_tests/heapprofile.sh + nohup bash /root/rss.sh + ./run config/indexes/n10_basic64K.json + tee -a /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/output.txt rm: cannot remove ‘gsi’: No such file or directory error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). Mon Sep 13 16:00:12 PDT 2021 RSS = Initializing write barrier = 8000 2021-09-13 16:00:22 [gsi-test] [] Total number of index instances 10 2021-09-13 16:00:22 [gsi-test] [] Local MemQuota: -------------- GSI-Plasma Test Config ------------------ {"RunTime":"3h","MemQuota":"5gb","QuotaTuner":true,"QuotaTunerMin":20,"QuotaTunerMax":50,"Verify":true,"VerifyTime":"1m","RecoveryInterval":"210m","SuppressLogs":false,"StopAndVerify":true,"StopAndVerifyTime":"5m","MemoryEvaluationMinimum":16777216,"MemoryEvaluationThreshold":1.6,"ActiveKeysConsistencyThreshold":0.85} -------------- GSI-Plasma Default Config ------------------ {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":null,"IdxRunTime":"120m","Load":0,"Workload":null,"Shared":true,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} -------------- GSI-Plasma Index Config ------------------ {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_0","IdxRunTime":"120m","Load":0,"Workload":"config/workload/01_ru_workload.json","Shared":false,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_1","IdxRunTime":"120m","Load":0,"Workload":"config/workload/01_ru_workload.json","Shared":false,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_2","IdxRunTime":"120m","Load":0,"Workload":"config/workload/01_ru_workload.json","Shared":true,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_3","IdxRunTime":"120m","Load":0,"Workload":"config/workload/01_ru_workload.json","Shared":true,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_4","IdxRunTime":"120m","Load":0,"Workload":"config/workload/02_rh_workload.json","Shared":false,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_5","IdxRunTime":"120m","Load":0,"Workload":"config/workload/02_rh_workload.json","Shared":true,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_6","IdxRunTime":"120m","Load":0,"Workload":"config/workload/02_rh_workload.json","Shared":true,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_7","IdxRunTime":"120m","Load":0,"Workload":"config/workload/03_ruc_workload.json","Shared":false,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1000},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_8","IdxRunTime":"120m","Load":0,"Workload":"config/workload/03_ruc_workload.json","Shared":true,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1000},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_9","IdxRunTime":"120m","Load":0,"Workload":"config/workload/03_ruc_workload.json","Shared":true,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1000},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} 2021-09-13 16:00:22 [gsi-test] [] Set global quota: 5368709120 2021-09-13 16:00:22 [gsi-test] [] The runtime is 10800 seconds in 3h0m0s. 2021-09-13 16:00:22 [gsi-test] [] Start Time: 2021-09-13 16:00:22.101929555 -0700 PDT m=+0.023645804 End Time: 2021-09-13 19:00:22.101929555 -0700 PDT m=+10800.023645804 2021-09-13 16:00:22 [gsi-test] [0] Init index instance %d with clean up Destory instances matching prefix /data/gsi_testdir/devel_0_0 in /data/gsi_testdir ... 2021-09-13 16:00:22 [plasma] [i] Start shard recovery from /data/gsi_testdir/shards2021-09-13 16:00:22 [plasma] [i] Directory /data/gsi_testdir/shards does not exist. Skip shard recovery. No instances to destroy matching prefix /data/gsi_testdir/devel_0_0 2021-09-13 16:00:22 [gsi-test] [0] RemoveAll(/data/gsi_testdir/devel_0_0) 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard Created Successfully 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : metadata saved successfully 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner initialized 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_0_0/docIndex 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCleaner initialized for recovery 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_0_0/docIndex/recovery 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Map plasma instance (/data/gsi_testdir/devel_0_0/docIndex) to LSS (/data/gsi_testdir/devel_0_0/docIndex) and RecoveryLSS (/data/gsi_testdir/devel_0_0/docIndex/recovery) 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Starting recovery. Recovery log [/data/gsi_testdir/devel_0_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_0_0/docIndex], Shared [false] 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoverFromHeaderReplay: Begin recovering from recovery log, headOffset [0] tailOffset [0] 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Done recovering from recovery log, replayOffset [0] took [136.177µs] 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Done recovery. Recovery log [/data/gsi_testdir/devel_0_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_0_0/docIndex], Shared [false]. Built [0] plasmas, took [158.154µs] 2021-09-13 16:00:22 [plasma] [i] Plasma: doInit: data UsedSpace 0 recovery UsedSpace 0 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Assign plasmaId 1 to instance /data/gsi_testdir/devel_0_0/docIndex 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Add instance /data/gsi_testdir/devel_0_0/docIndex to Shard /data/gsi_testdir/shards/shard1 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Add instance /data/gsi_testdir/devel_0_0/docIndex to LSS /data/gsi_testdir/devel_0_0/docIndex 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper started 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Instance added to swapper : /data/gsi_testdir/shards/shard1 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : instance /data/gsi_testdir/devel_0_0/docIndex started 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 16:00:22 [gsi-test] [1] Init index instance %d with clean up Destory instances matching prefix /data/gsi_testdir/devel_1_0 in /data/gsi_testdir ... No instances to destroy matching prefix /data/gsi_testdir/devel_1_0 2021-09-13 16:00:22 [gsi-test] [1] RemoveAll(/data/gsi_testdir/devel_1_0) 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Shard Created Successfully 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : metadata saved successfully 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner initialized 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_1_0/docIndex 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCleaner initialized for recovery 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_1_0/docIndex/recovery 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Map plasma instance (/data/gsi_testdir/devel_1_0/docIndex) to LSS (/data/gsi_testdir/devel_1_0/docIndex) and RecoveryLSS (/data/gsi_testdir/devel_1_0/docIndex/recovery) 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Shard.doRecovery: Starting recovery. Recovery log [/data/gsi_testdir/devel_1_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_1_0/docIndex], Shared [false] 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoverFromHeaderReplay: Begin recovering from recovery log, headOffset [0] tailOffset [0] 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Shard.doRecovery: Done recovering from recovery log, replayOffset [0] took [81.051µs] 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Shard.doRecovery: Done recovery. Recovery log [/data/gsi_testdir/devel_1_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_1_0/docIndex], Shared [false]. Built [0] plasmas, took [160.463µs] 2021-09-13 16:00:22 [plasma] [i] Plasma: doInit: data UsedSpace 0 recovery UsedSpace 0 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Assign plasmaId 1 to instance /data/gsi_testdir/devel_1_0/docIndex 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Add instance /data/gsi_testdir/devel_1_0/docIndex to Shard /data/gsi_testdir/shards/shard2 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Add instance /data/gsi_testdir/devel_1_0/docIndex to LSS /data/gsi_testdir/devel_1_0/docIndex 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper started 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Instance added to swapper : /data/gsi_testdir/shards/shard2 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : instance /data/gsi_testdir/devel_1_0/docIndex started 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 16:00:22 [gsi-test] [2] Init index instance %d with clean up Destory instances matching prefix /data/gsi_testdir/devel_2_0 in /data/gsi_testdir ... No instances to destroy matching prefix /data/gsi_testdir/devel_2_0 2021-09-13 16:00:22 [gsi-test] [2] RemoveAll(/data/gsi_testdir/devel_2_0) 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Shard Created Successfully 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : metadata saved successfully 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner initialized 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : LSSCtx Created Successfully. Path=/data/gsi_testdir/shards/shard3/data 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCleaner initialized for recovery 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : LSSCtx Created Successfully. Path=/data/gsi_testdir/shards/shard3/data/recovery 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Map plasma instance (/data/gsi_testdir/devel_2_0/docIndex) to LSS (/data/gsi_testdir/shards/shard3/data) and RecoveryLSS (/data/gsi_testdir/shards/shard3/data/recovery) 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Shard.doRecovery: Starting recovery. Recovery log [/data/gsi_testdir/shards/shard3/data/recovery], Data log [/data/gsi_testdir/shards/shard3/data], Shared [true] 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoverFromHeaderReplay: Begin recovering from recovery log, headOffset [0] tailOffset [0] 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Shard.doRecovery: Done recovering from recovery log, replayOffset [0] took [156.803µs] 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : recoverFromDataReplay: Begin recovering from data log, headOffset [0] tailOffset [0] replayOffset [0] 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Shard.doRecovery: Done recovering from data and recovery log, took [204.342µs] 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Shard.doRecovery: Done recovery. Recovery log [/data/gsi_testdir/shards/shard3/data/recovery], Data log [/data/gsi_testdir/shards/shard3/data], Shared [true]. Built [0] plasmas, took [224.159µs] 2021-09-13 16:00:22 [plasma] [i] Plasma: doInit: data UsedSpace 0 recovery UsedSpace 0 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Assign plasmaId 1 to instance /data/gsi_testdir/devel_2_0/docIndex 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_2_0/docIndex to Shard /data/gsi_testdir/shards/shard3 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_2_0/docIndex to LSS /data/gsi_testdir/shards/shard3/data 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper started 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Instance added to swapper : /data/gsi_testdir/shards/shard3 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : instance /data/gsi_testdir/devel_2_0/docIndex started 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 16:00:22 [gsi-test] [3] Init index instance %d with clean up Destory instances matching prefix /data/gsi_testdir/devel_3_0 in /data/gsi_testdir ... No instances to destroy matching prefix /data/gsi_testdir/devel_3_0 2021-09-13 16:00:22 [gsi-test] [3] RemoveAll(/data/gsi_testdir/devel_3_0) 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : metadata saved successfully 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Map plasma instance (/data/gsi_testdir/devel_3_0/docIndex) to LSS (/data/gsi_testdir/shards/shard3/data) and RecoveryLSS (/data/gsi_testdir/shards/shard3/data/recovery) 2021-09-13 16:00:22 [plasma] [i] Plasma: doInit: data UsedSpace 0 recovery UsedSpace 0 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Assign plasmaId 2 to instance /data/gsi_testdir/devel_3_0/docIndex 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_3_0/docIndex to Shard /data/gsi_testdir/shards/shard3 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_3_0/docIndex to LSS /data/gsi_testdir/shards/shard3/data 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Instance added to swapper : /data/gsi_testdir/shards/shard3 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : instance /data/gsi_testdir/devel_3_0/docIndex started 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 16:00:22 [gsi-test] [4] Init index instance %d with clean up Destory instances matching prefix /data/gsi_testdir/devel_4_0 in /data/gsi_testdir ... No instances to destroy matching prefix /data/gsi_testdir/devel_4_0 2021-09-13 16:00:22 [gsi-test] [4] RemoveAll(/data/gsi_testdir/devel_4_0) 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : metadata saved successfully 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner initialized 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_4_0/docIndex 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCleaner initialized for recovery 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_4_0/docIndex/recovery 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Map plasma instance (/data/gsi_testdir/devel_4_0/docIndex) to LSS (/data/gsi_testdir/devel_4_0/docIndex) and RecoveryLSS (/data/gsi_testdir/devel_4_0/docIndex/recovery) 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Starting recovery. Recovery log [/data/gsi_testdir/devel_4_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_4_0/docIndex], Shared [false] 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoverFromHeaderReplay: Begin recovering from recovery log, headOffset [0] tailOffset [0] 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Done recovering from recovery log, replayOffset [0] took [79.064µs] 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Done recovery. Recovery log [/data/gsi_testdir/devel_4_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_4_0/docIndex], Shared [false]. Built [0] plasmas, took [98.63µs] 2021-09-13 16:00:22 [plasma] [i] Plasma: doInit: data UsedSpace 0 recovery UsedSpace 0 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Assign plasmaId 2 to instance /data/gsi_testdir/devel_4_0/docIndex 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Add instance /data/gsi_testdir/devel_4_0/docIndex to Shard /data/gsi_testdir/shards/shard1 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Add instance /data/gsi_testdir/devel_4_0/docIndex to LSS /data/gsi_testdir/devel_4_0/docIndex 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Instance added to swapper : /data/gsi_testdir/shards/shard1 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : instance /data/gsi_testdir/devel_4_0/docIndex started 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 16:00:22 [gsi-test] [5] Init index instance %d with clean up Destory instances matching prefix /data/gsi_testdir/devel_5_0 in /data/gsi_testdir ... No instances to destroy matching prefix /data/gsi_testdir/devel_5_0 2021-09-13 16:00:22 [gsi-test] [5] RemoveAll(/data/gsi_testdir/devel_5_0) 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : metadata saved successfully 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Map plasma instance (/data/gsi_testdir/devel_5_0/docIndex) to LSS (/data/gsi_testdir/shards/shard3/data) and RecoveryLSS (/data/gsi_testdir/shards/shard3/data/recovery) 2021-09-13 16:00:22 [plasma] [i] Plasma: doInit: data UsedSpace 0 recovery UsedSpace 0 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Assign plasmaId 3 to instance /data/gsi_testdir/devel_5_0/docIndex 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_5_0/docIndex to Shard /data/gsi_testdir/shards/shard3 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_5_0/docIndex to LSS /data/gsi_testdir/shards/shard3/data 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Instance added to swapper : /data/gsi_testdir/shards/shard3 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : instance /data/gsi_testdir/devel_5_0/docIndex started 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 16:00:22 [gsi-test] [6] Init index instance %d with clean up Destory instances matching prefix /data/gsi_testdir/devel_6_0 in /data/gsi_testdir ... No instances to destroy matching prefix /data/gsi_testdir/devel_6_0 2021-09-13 16:00:22 [gsi-test] [6] RemoveAll(/data/gsi_testdir/devel_6_0) 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : metadata saved successfully 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Map plasma instance (/data/gsi_testdir/devel_6_0/docIndex) to LSS (/data/gsi_testdir/shards/shard3/data) and RecoveryLSS (/data/gsi_testdir/shards/shard3/data/recovery) 2021-09-13 16:00:22 [plasma] [i] Plasma: doInit: data UsedSpace 0 recovery UsedSpace 0 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Assign plasmaId 4 to instance /data/gsi_testdir/devel_6_0/docIndex 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_6_0/docIndex to Shard /data/gsi_testdir/shards/shard3 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_6_0/docIndex to LSS /data/gsi_testdir/shards/shard3/data 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Instance added to swapper : /data/gsi_testdir/shards/shard3 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : instance /data/gsi_testdir/devel_6_0/docIndex started 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 16:00:22 [gsi-test] [7] Init index instance %d with clean up Destory instances matching prefix /data/gsi_testdir/devel_7_0 in /data/gsi_testdir ... No instances to destroy matching prefix /data/gsi_testdir/devel_7_0 2021-09-13 16:00:22 [gsi-test] [7] RemoveAll(/data/gsi_testdir/devel_7_0) 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : metadata saved successfully 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner initialized 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_7_0/docIndex 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCleaner initialized for recovery 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_7_0/docIndex/recovery 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Map plasma instance (/data/gsi_testdir/devel_7_0/docIndex) to LSS (/data/gsi_testdir/devel_7_0/docIndex) and RecoveryLSS (/data/gsi_testdir/devel_7_0/docIndex/recovery) 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Starting recovery. Recovery log [/data/gsi_testdir/devel_7_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_7_0/docIndex], Shared [false] 2021-09-13 16:00:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoverFromHeaderReplay: Begin recovering from recovery log, headOffset [0] tailOffset [0] 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Done recovering from recovery log, replayOffset [0] took [85.807µs] 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Done recovery. Recovery log [/data/gsi_testdir/devel_7_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_7_0/docIndex], Shared [false]. Built [0] plasmas, took [105.1µs] 2021-09-13 16:00:22 [plasma] [i] Plasma: doInit: data UsedSpace 0 recovery UsedSpace 0 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Assign plasmaId 3 to instance /data/gsi_testdir/devel_7_0/docIndex 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Add instance /data/gsi_testdir/devel_7_0/docIndex to Shard /data/gsi_testdir/shards/shard1 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Add instance /data/gsi_testdir/devel_7_0/docIndex to LSS /data/gsi_testdir/devel_7_0/docIndex 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Instance added to swapper : /data/gsi_testdir/shards/shard1 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : instance /data/gsi_testdir/devel_7_0/docIndex started 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 16:00:22 [gsi-test] [8] Init index instance %d with clean up Destory instances matching prefix /data/gsi_testdir/devel_8_0 in /data/gsi_testdir ... No instances to destroy matching prefix /data/gsi_testdir/devel_8_0 2021-09-13 16:00:22 [gsi-test] [8] RemoveAll(/data/gsi_testdir/devel_8_0) 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : metadata saved successfully 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Map plasma instance (/data/gsi_testdir/devel_8_0/docIndex) to LSS (/data/gsi_testdir/shards/shard3/data) and RecoveryLSS (/data/gsi_testdir/shards/shard3/data/recovery) 2021-09-13 16:00:22 [plasma] [i] Plasma: doInit: data UsedSpace 0 recovery UsedSpace 0 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Assign plasmaId 5 to instance /data/gsi_testdir/devel_8_0/docIndex 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_8_0/docIndex to Shard /data/gsi_testdir/shards/shard3 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_8_0/docIndex to LSS /data/gsi_testdir/shards/shard3/data 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Instance added to swapper : /data/gsi_testdir/shards/shard3 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : instance /data/gsi_testdir/devel_8_0/docIndex started 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 16:00:22 [gsi-test] [9] Init index instance %d with clean up Destory instances matching prefix /data/gsi_testdir/devel_9_0 in /data/gsi_testdir ... No instances to destroy matching prefix /data/gsi_testdir/devel_9_0 2021-09-13 16:00:22 [gsi-test] [9] RemoveAll(/data/gsi_testdir/devel_9_0) 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : metadata saved successfully 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Map plasma instance (/data/gsi_testdir/devel_9_0/docIndex) to LSS (/data/gsi_testdir/shards/shard3/data) and RecoveryLSS (/data/gsi_testdir/shards/shard3/data/recovery) 2021-09-13 16:00:22 [plasma] [i] Plasma: doInit: data UsedSpace 0 recovery UsedSpace 0 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Assign plasmaId 6 to instance /data/gsi_testdir/devel_9_0/docIndex 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_9_0/docIndex to Shard /data/gsi_testdir/shards/shard3 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_9_0/docIndex to LSS /data/gsi_testdir/shards/shard3/data 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Instance added to swapper : /data/gsi_testdir/shards/shard3 2021-09-13 16:00:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : instance /data/gsi_testdir/devel_9_0/docIndex started 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 16:00:22 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 16:00:22 [gsi-test] [9] Instance... START 2021-09-13 16:00:22 [gsi-test] [9] Starting job for index instance 2021-09-13 16:00:22 [gsi-test] [4] Instance... START 2021-09-13 16:00:22 [gsi-test] [9] CreateSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [4] Starting job for index instance 2021-09-13 16:00:22 [gsi-test] [6] Instance... START 2021-09-13 16:00:22 [gsi-test] [6] Starting job for index instance 2021-09-13 16:00:22 [gsi-test] [9] DeleteSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [6] CreateSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [4] CreateSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [6] DeleteSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [4] DeleteSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [7] Instance... START 2021-09-13 16:00:22 [gsi-test] [7] Starting job for index instance 2021-09-13 16:00:22 [gsi-test] [9:15] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [9:15] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [7] CreateSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [9:15] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2] Instance... START 2021-09-13 16:00:22 [gsi-test] [2] Starting job for index instance 2021-09-13 16:00:22 [gsi-test] [4:0] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [4:0] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1] Instance... START 2021-09-13 16:00:22 [gsi-test] [1] Starting job for index instance 2021-09-13 16:00:22 [gsi-test] [4:0] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2] CreateSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [1] CreateSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [6:15] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:15] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:15] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [5] Instance... START 2021-09-13 16:00:22 [gsi-test] [5] Starting job for index instance 2021-09-13 16:00:22 [gsi-test] [5] CreateSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [5] DeleteSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [7] DeleteSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [9:6] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [9:6] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:6] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:15] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0] Instance... START 2021-09-13 16:00:22 [gsi-test] [7:15] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [7:15] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0] Starting job for index instance 2021-09-13 16:00:22 [gsi-test] [5:15] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [5:15] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:15] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:1] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [7:1] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [7:1] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:0] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0] CreateSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [6:1] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:1] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:1] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [6:0] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:0] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:0] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [1] DeleteSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [6:2] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:2] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:2] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [6:3] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:3] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:3] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [9:2] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [9:2] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:2] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:1] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [4:1] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [4:1] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3] Instance... START 2021-09-13 16:00:22 [gsi-test] [3] Starting job for index instance 2021-09-13 16:00:22 [gsi-test] [3] CreateSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [4:15] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [4:15] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [4:15] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:2] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [4:2] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [4:2] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:0] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1:15] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:15] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1:15] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [1:2] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:2] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1:2] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:0] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2] DeleteSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [3] DeleteSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [1:0] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:0] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1:0] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2:15] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:15] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [2:15] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3:15] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:15] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [3:15] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:8] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [4:8] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [4:8] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [9:3] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [9:3] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:3] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:3] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [4:3] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [4:3] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0] DeleteSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [8] Instance... START 2021-09-13 16:00:22 [gsi-test] [7:3] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [7:3] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [7:3] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [6:5] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [4:10] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [9:10] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [4:10] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:10] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:10] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:10] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [6:5] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:8] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:5] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:11] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [7:4] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [4:12] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:7] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:7] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:7] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [9:7] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [9:7] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:7] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:6] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [7:6] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [7:6] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:5] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [7:5] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [7:5] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [6:4] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:1] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:1] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1:1] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2:5] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:6] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:6] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [4:11] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8] Starting job for index instance 2021-09-13 16:00:22 [gsi-test] [7:4] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:4] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [9:4] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:4] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [9:9] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [9:9] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:9] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:11] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2:5] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [2:5] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [9:8] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:11] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [9:11] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:11] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:4] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [9:8] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:2] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [7:2] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [7:2] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:12] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [4:12] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [6:4] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:4] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [9:1] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [9:1] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:1] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [6:6] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2:1] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:1] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [2:1] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [5:13] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [5:13] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:13] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [1:9] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:9] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1:9] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2:2] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:2] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [2:2] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:5] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [4:5] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [4:5] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [9:13] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:4] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [9:12] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [9:12] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:13] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:12] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:9] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [4:9] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [4:9] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [9:14] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [9:14] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:14] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [9:0] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [9:0] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:0] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2:0] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:0] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [2:0] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [9:13] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2:4] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [2:4] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:13] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [4:13] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [4:13] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2:3] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:3] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [2:3] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8] CreateSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [8] DeleteSnapshot() - START 2021-09-13 16:00:22 [gsi-test] [3:2] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:2] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [3:2] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8:15] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [8:15] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:15] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8:7] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [8:7] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:7] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8:0] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [8:0] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:13] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [8:13] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:1] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [8:13] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8:1] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:1] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8:0] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8:12] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [8:12] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:12] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:14] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [4:14] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [4:14] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:7] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [4:7] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [4:7] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [6:9] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:9] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:9] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8:2] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [8:2] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:2] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2:6] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:6] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:10] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:6] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [6:10] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:10] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:4] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:11] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:11] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:11] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:4] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [4:4] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3:1] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:1] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [3:1] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [6:12] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:12] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:12] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [6:13] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:13] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:13] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4:6] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [4:6] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [4:6] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [4] CreateRecoveryPoint() - START 2021-09-13 16:00:22 [gsi-test] [4] Rollback() - START 2021-09-13 16:00:22 [gsi-test] [4] Generation() - START 2021-09-13 16:00:22 [gsi-test] [6:8] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [6:8] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:8] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8:3] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [8:3] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:3] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8:11] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [8:11] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:11] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8:4] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [8:4] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:4] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:8] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [7:8] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [7:8] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:12] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [7:12] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [7:7] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [7:7] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [7:7] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:1] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0:1] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [0:1] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:12] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:4] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0:4] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [0:7] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0:8] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0:8] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [0:8] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:7] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [0:3] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0:3] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [0:3] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:4] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:13] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0:11] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0:11] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [0:11] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:7] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:9] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [7:9] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [7:9] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:12] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0:12] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [0:12] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:10] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [7:10] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [7:10] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:13] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [0:13] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:14] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [7:14] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [7:14] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:2] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0:2] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [0:2] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:10] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0:15] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:11] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:11] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [2:11] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2:9] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:9] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [2:9] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2:10] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:10] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [2:10] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2:8] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:8] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [2:8] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [5:7] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [5:7] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:7] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [5:0] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [5:0] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:0] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3:0] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:0] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:2] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:0] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [5:2] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:3] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [5:3] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:3] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3:4] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:4] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [3:4] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3:3] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:3] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [3:3] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [5:2] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8:10] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [8:10] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:10] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3:9] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:9] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [3:9] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3:5] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:5] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:1] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:5] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3:6] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:6] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [3:6] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [5:1] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:4] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [5:4] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:4] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [5:1] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8:8] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0:6] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [8:14] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [8:6] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0:9] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [8:5] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [8:9] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [7:11] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [5:14] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [5:14] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:14] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [1:10] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:10] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:5] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:10] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [5:5] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [7:11] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:14] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:14] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3:10] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:10] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [3:10] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:15] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [0:15] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [5:5] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7:11] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [5:12] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [5:12] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:12] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:0] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0:0] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [0:0] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [1:8] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:8] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1:8] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [1:3] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:3] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1:3] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [1:4] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:4] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1:4] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [1:5] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:5] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1:5] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [1:6] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:6] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1:7] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:6] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [1:7] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1:11] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:11] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1:11] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [1:7] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [1:13] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:12] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:14] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:7] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:14] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:14] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1:14] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [6:14] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6:14] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [6] CreateRecoveryPoint() - START 2021-09-13 16:00:22 [gsi-test] [1] CreateRecoveryPoint() - START 2021-09-13 16:00:22 [gsi-test] [1] Rollback() - START 2021-09-13 16:00:22 [gsi-test] [9:5] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [1] Generation() - START 2021-09-13 16:00:22 [gsi-test] [9:5] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1:12] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [9:5] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [1:12] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [9] CreateRecoveryPoint() - START 2021-09-13 16:00:22 [gsi-test] [6] Rollback() - START 2021-09-13 16:00:22 [gsi-test] [0:14] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0:14] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [6] Generation() - START 2021-09-13 16:00:22 [gsi-test] [0:14] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [1:13] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [0:5] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [0:5] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [1:13] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:5] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0] CreateRecoveryPoint() - START 2021-09-13 16:00:22 [gsi-test] [0] Rollback() - START 2021-09-13 16:00:22 [gsi-test] [0] Generation() - START 2021-09-13 16:00:22 [gsi-test] [8:8] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:8] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:6] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [0:6] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8:6] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:6] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:9] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [0:9] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [9] Rollback() - START 2021-09-13 16:00:22 [gsi-test] [2:7] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:7] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [2:7] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [9] Generation() - START 2021-09-13 16:00:22 [gsi-test] [3:7] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [3:7] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8:5] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:5] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [0:10] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [0:10] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8:9] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [8:9] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [8] CreateRecoveryPoint() - START 2021-09-13 16:00:22 [gsi-test] [8] Rollback() - START 2021-09-13 16:00:22 [gsi-test] [8] Generation() - START 2021-09-13 16:00:22 [gsi-test] [7:13] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [7:13] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [7:13] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [7] CreateRecoveryPoint() - START 2021-09-13 16:00:22 [gsi-test] [7] Rollback() - START 2021-09-13 16:00:22 [gsi-test] [7] Generation() - START 2021-09-13 16:00:22 [gsi-test] [2:13] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:13] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [2:13] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [5:6] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [5:6] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:6] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3:11] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:11] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [3:11] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2:12] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:12] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [2:12] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3:8] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:8] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [3:8] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3:14] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:14] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [3:14] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [5:11] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [5:11] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:11] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [5:8] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [5:8] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:8] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3:13] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [3:13] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [3:13] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3:12] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [5:10] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [5:10] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:10] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3] CreateRecoveryPoint() - START 2021-09-13 16:00:22 [gsi-test] [3] Rollback() - START 2021-09-13 16:00:22 [gsi-test] [5:9] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [5:9] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [5:9] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [3:12] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [3:12] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [5] CreateRecoveryPoint() - START 2021-09-13 16:00:22 [gsi-test] [5] Rollback() - START 2021-09-13 16:00:22 [gsi-test] [5] Generation() - START 2021-09-13 16:00:22 [gsi-test] [3] Generation() - START 2021-09-13 16:00:22 [gsi-test] [2:14] Worker.Run() - START 2021-09-13 16:00:22 [gsi-test] [2:14] Worker.Run() - LOAD 2021-09-13 16:00:22 [gsi-test] [2:14] Worker.Run() - WORKLOAD 2021-09-13 16:00:22 [gsi-test] [2] CreateRecoveryPoint() - START 2021-09-13 16:00:22 [gsi-test] [2] Rollback() - START 2021-09-13 16:00:22 [gsi-test] [2] Generation() - START 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : all deamons started 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : all deamons started 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : all deamons started 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : all deamons started 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : all deamons started 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : all deamons started 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : all deamons started 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:00:42 [gsi-test] [] FullStats... START 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:00:42 [gsi-test] [] Stop and verify... START 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:00:42 [gsi-test] [] GSI-Plasma Current Time 2021-09-13 16:00:42.18578167 -0700 PDT m=+20.107497553 2021-09-13 16:00:42 [gsi-test] [] QuotaTuner... START 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:00:42 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:00:42 [gsi-test] [] MemoryStats... START 2021-09-13 16:00:42 [gsi-test] [] ShortStats... START Mon Sep 13 16:01:12 PDT 2021 RSS = 954752 2021-09-13T16:01:12.336-07:00 [Info] memstats {"Alloc":248250712, "TotalAlloc":34754982536, "Sys":352526584, "Lookups":0, "Mallocs":1114315,"Frees":1099238, "HeapAlloc":248250712, "HeapSys":331874304, "HeapIdle":81502208, "HeapInuse":250372096,"HeapReleased":47464448, "HeapObjects":15077,"GCSys":11143168, "LastGC":1631574072335409156,"PauseTotalNs":3270013330, "PauseNs":[2672273, 5141474, 6582049, 7335155, 11359177, 103583839, 74582801, 11252407, 15323247, 7741928, 11654831, 8680395, 14999185, 17752629, 13105565, 17919603, 21464832, 17159775, 13271412, 26584250, 18100235, 3006099, 22974498, 18817612, 24432558, 21697849, 19782361, 23684070, 25062623, 21265827, 24939439, 18878431, 31867651, 25734258, 29830399, 27103229, 23202242, 30142303, 32629418, 4855213, 29419758, 9214296, 15139115, 1817499, 18570674, 16142957, 30631977, 16062169, 35871212, 14052611, 1156247, 19799218, 101625661, 19904202, 2488651, 786008, 18061954, 3522098, 11693654, 16157222, 23616033, 19352511, 870081, 7922288, 15671580, 33033168, 7263723, 11468786, 10830158, 11143619, 1087318, 3043216, 2772504, 11521709, 20475240, 11460510, 18766294, 2062950, 7628284, 6742919, 1547409, 6513979, 8854636, 4178025, 1201521, 1566658, 754474, 1746685, 1578697, 1071450, 735930, 1776916, 1756991, 1200436, 1685326, 1224317, 1390042, 1880255, 1873420, 580233, 1556584, 1395734, 1398345, 1568061, 861539, 845904, 1414781, 1282118, 2526072, 894100, 1447951, 1152118, 1388966, 1778068, 900562, 1298633, 585232, 1008737, 2014378, 2344110, 1112292, 2176292, 990330, 1408695, 1953817, 1293811, 1065061, 1073170, 1481863, 814565, 780176, 1634970, 1357281, 1147565, 1048917, 1753684, 1045336, 1442320, 1363997, 2434638, 1222629, 1252389, 1479968, 1303298, 1257891, 1778778, 1454771, 1533181, 1592009, 1107072, 1018139, 1575683, 1738830, 1233862, 2002058, 1505895, 1397602, 813238, 1381921, 545691, 1044367, 2563788, 3113554, 5617803, 5856295, 4582643, 2186181, 3750557, 7313348, 3943290, 6944703, 6661070, 11011844, 5881635, 10510025, 3255269, 19200421, 15950354, 14572635, 17103619, 15319461, 17477180, 20363131, 19846296, 18521796, 14256383, 12434723, 16810621, 22637264, 12553280, 26578528, 9672302, 7302709, 16782971, 13057356, 17229585, 19605252, 19758648, 6986893, 3893596, 2477018, 16638711, 9265834, 22170466, 33356199, 3002043, 28342996, 27016758, 8567116, 6016754, 21271838, 9529213, 1605412, 878497, 728641, 1108501, 985789, 501505, 1327331, 1264759, 3159373, 5928967, 4966069, 6574511, 4898891, 10471334, 8905176, 7064597, 6848492, 14750021, 11909144, 11512380, 11752019, 17974075, 22900101, 22792342, 21223748, 30635341, 25912301, 33807600, 25487368, 31270777, 39644726, 26079759, 33521002, 19758888, 23814167, 18370605, 20792138, 8511047, 1610668, 1392085, 1050474, 1467194, 1797417, 2370679], "NumGC":596} Allocated: 801972352, active: 801988608, metadata: 10105152 (n_thp 0), resident: 864550912, mapped: 893300736, retained: 763449344 2021-09-13T16:01:42.656-07:00 [Info] memstats {"Alloc":303360456, "TotalAlloc":84928605384, "Sys":421800184, "Lookups":0, "Mallocs":2744191,"Frees":2726929, "HeapAlloc":303360456, "HeapSys":398589952, "HeapIdle":92987392, "HeapInuse":305602560,"HeapReleased":91332608, "HeapObjects":17262,"GCSys":13344768, "LastGC":1631574102616678264,"PauseTotalNs":4607795719, "PauseNs":[1104730, 2014038, 1777007, 784297, 1384816, 1300238, 1229088, 2284239, 2033887, 1623223, 1307776, 1165953, 1590104, 1389896, 1310776, 644625, 1000081, 1155268, 1276194, 1839466, 894537, 1689307, 1576441, 1718130, 1703741, 1342463, 2225141, 1454852, 758800, 1449874, 1226201, 985437, 1512330, 903040, 604921, 791411, 1856914, 1519052, 920307, 1313652, 1497759, 1359656, 1557649, 1207064, 1640626, 1079636, 4443557, 1704256, 1557151, 881649, 2705668, 1018734, 1170006, 1184326, 1700714, 1282036, 1754526, 1176948, 1457499, 1369855, 2417045, 879921, 1600282, 1259381, 1521382, 1652036, 1664294, 1509757, 1083146, 1541480, 1885268, 1938570, 1894191, 1185700, 1377450, 594762, 1074168, 1966175, 1629094, 681955, 2346770, 1360338, 1144291, 1508864, 67067770, 2279827, 1674032, 1193954, 1388196, 550845, 1735940, 754401, 1654928, 1313392, 948887, 1237413, 1355091, 1315258, 2290330, 2673520, 1094869, 1690965, 1484046, 1175573, 1309463, 1233738, 927238, 1214302, 1457645, 1011469, 1492823, 1095408, 1068686, 1253495, 1021521, 1855080, 1694165, 770675, 1108474, 1190077, 1279560, 1126476, 1498378, 1556534, 2060951, 1401340, 1208555, 3056183, 1472043, 1713478, 1034177, 1065539, 1410751, 721138, 866363, 1333399, 1444247, 944709, 993358, 1611340, 1456120, 1429895, 1347895, 1393332, 1554980, 1667113, 680003, 1431144, 1803526, 1470442, 1221119, 2192332, 1496797, 1418835, 1362109, 1256574, 1534317, 1694036, 1331917, 1276073, 1344360, 1925310, 1112520, 1490137, 1231061, 1590882, 922298, 1042973, 1844844, 1493995, 1701954, 1628485, 1732174, 4274431, 57179060, 1348184, 1027518, 1108902, 1255092, 894942, 454912, 1101956, 1520111, 983860, 1415370, 1058364, 2641100, 1219286, 1280715, 1165269, 2159403, 2920517, 1342013, 2002388, 1541750, 1441452, 1001631, 657868, 1530254, 1419279, 1710228, 856445, 1448871, 1241361, 2598743, 1311835, 1612564, 1913535, 1312425, 2117131, 1195763, 979204, 1366969, 1227011, 1072643, 1555504, 545274, 1160663, 2158098, 1251129, 1917821, 1664617, 1069434, 1273455, 1862392, 3973711, 1183487, 1138160, 1483520, 643550, 1613236, 1044179, 1160402, 1491465, 1466015, 1059569, 1276428, 1050802, 1320880, 1557067, 1104280, 619587, 694555, 1417240, 3220687, 1092670, 742371, 1030451, 673279, 2098357, 1603205, 1680266, 1248131, 1304821, 1845712, 1598799], "NumGC":1033} Allocated: 872079488, active: 872095744, metadata: 10131264 (n_thp 0), resident: 891858944, mapped: 907993088, retained: 748756992 2021-09-13 16:01:42 [ss-workload] instance inse% dele% read% 2021-09-13 16:01:42 [ss-workload] 0 16% 16% 66% 2021-09-13 16:01:42 [ss-workload] 1 16% 16% 66% 2021-09-13 16:01:42 [ss-workload] 2 16% 16% 66% 2021-09-13 16:01:42 [ss-workload] 3 16% 16% 66% 2021-09-13 16:01:42 [ss-workload] 4 3% 3% 92% 2021-09-13 16:01:42 [ss-workload] 5 3% 3% 92% 2021-09-13 16:01:42 [ss-workload] 6 3% 3% 92% 2021-09-13 16:01:42 [ss-workload] 7 19% 19% 61% 2021-09-13 16:01:42 [ss-workload] 8 19% 19% 61% 2021-09-13 16:01:42 [ss-workload] 9 19% 19% 61% 2021-09-13 16:01:42 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:01:42 [ss-diag] 0 3376 2952 0 0 1770 7 116019960 1 27802168 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:01:42 [ss-diag] 1 3686 3448 0 0 2054 8 134635592 1 15603152 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:01:42 [ss-diag] 2 3493 3140 0 0 1836 8 120346128 1 23142892 0 1.000000 0.840056 1.000000 0:0:0 2021-09-13 16:01:42 [ss-diag] 3 3330 2936 0 0 1746 7 114446808 1 25834784 0 1.000000 0.840056 1.000000 0:0:0 2021-09-13 16:01:42 [ss-diag] 4 13699 13169 0 0 1035 6 67842180 2 34814724 65588 1.000000 2.047861 1.000000 0:0:0 2021-09-13 16:01:42 [ss-diag] 5 12236 11724 0 0 778 5 50996344 2 33635612 65588 1.000000 0.840056 1.000000 0:0:0 2021-09-13 16:01:42 [ss-diag] 6 14215 13787 0 0 2315 7 151743620 2 28124172 65604 1.000000 0.840056 1.000000 0:0:0 2021-09-13 16:01:42 [ss-diag] 7 6559 6219 0 0 503 2 32970644 1 22291080 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:01:42 [ss-diag] 8 6545 6295 0 0 487 2 31921876 1 16389504 0 1.000000 0.840056 1.000000 0:0:0 2021-09-13 16:01:42 [ss-diag] 9 7036 6626 0 0 625 2 40967500 1 26878496 0 1.000000 0.840056 1.000000 0:0:0 Mon Sep 13 16:02:12 PDT 2021 RSS = 1558216 2021-09-13T16:02:12.938-07:00 [Info] memstats {"Alloc":235352632, "TotalAlloc":123688172432, "Sys":421800184, "Lookups":0, "Mallocs":4000213,"Frees":3984886, "HeapAlloc":235352632, "HeapSys":398426112, "HeapIdle":160800768, "HeapInuse":237625344,"HeapReleased":76439552, "HeapObjects":15327,"GCSys":13414400, "LastGC":1631574132930205064,"PauseTotalNs":6207724544, "PauseNs":[1168190, 992652, 922907, 1804251, 1136091, 1364905, 3193030, 1621891, 1432660, 1142718, 762942, 1541320, 1502157, 1369195, 57438703, 1394421, 1697087, 681355, 1704515, 976794, 811069, 1755724, 1361749, 1077002, 1209948, 629152, 805522, 496735, 1053536, 1875444, 1135641, 1246046, 2015551, 886122, 1586693, 1503775, 1489127, 1304077, 846176, 1148469, 321938, 1410425, 1406713, 529713, 1310597, 1716335, 1354522, 1264532, 1397650, 766400, 1408818, 1436856, 2204201, 2137266, 1687295, 980177, 1185336, 1785843, 1706248, 1514951, 1050040, 1159350, 1043006, 1637515, 881243, 1297659, 795688, 1191619, 1640173, 1257896, 2021053, 1735945, 1563184, 1507927, 1341443, 1673123, 1140756, 875965, 1466187, 710057, 1201882, 1279628, 1372782, 1234057, 1805393, 1414701, 987336, 1509492, 2064842, 1538391, 1635146, 1280408, 431914, 1609609, 1796419, 1776286, 935184, 1909647, 2213390, 1417212, 1617702, 11292493, 39512356, 14729883, 12920788, 20839273, 10913857, 17854829, 23941709, 15899118, 28032545, 33503640, 26777612, 29460288, 42209840, 28494422, 34054475, 37273471, 40740631, 30151913, 8259091, 9324687, 13174029, 13862665, 12218478, 20172648, 14160454, 20081742, 86365295, 33319935, 33636365, 42451180, 29007514, 35037175, 45672913, 34887772, 40329133, 35291015, 41886203, 27402063, 43871076, 1426625, 860143, 2946178, 2215090, 1052360, 1022956, 827582, 1299657, 1588531, 1259854, 639935, 938503, 1859631, 1360886, 1037718, 1564438, 2422437, 1582700, 1668928, 1246058, 1220926, 1162358, 1064420, 1091426, 1345247, 1880289, 1229857, 1487160, 1181710, 1014010, 1124027, 1257687, 1369763, 4101679, 1983294, 2377164, 1910559, 795549, 844818, 810359, 1496943, 920455, 3267131, 1621306, 1071645, 1889293, 1500724, 1901562, 1472509, 1258531, 1114254, 1821823, 1939210, 2182813, 1403706, 1108319, 1389548, 2741569, 1426368, 1440125, 1353992, 1181085, 854119, 1616974, 2261612, 1438211, 1079263, 1537194, 1510540, 2904259, 1374443, 1956276, 1099592, 1233728, 1489544, 1106173, 1042021, 1004907, 1244394, 1338688, 1727699, 1473657, 848805, 2071912, 865008, 1981100, 881498, 1714429, 781748, 1271870, 725368, 1182286, 1203332, 1287261, 2840952, 1016421, 617361, 1380112, 1358126, 446965, 1239304, 1312168, 1262259, 1366723, 699008, 1611611, 1792595, 1376298, 981262, 2821095, 1852219, 1331293, 1370295, 1259323, 1976489], "NumGC":1362} Allocated: 1656478592, active: 1656500224, metadata: 12312960 (n_thp 0), resident: 1667653632, mapped: 1681666048, retained: 1417924608 2021-09-13 16:02:35 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: starting... frag 99, data: 49152, used: 17092608 log:(0 - 17092608) head:0 2021-09-13 16:02:35 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: completed... frag 99, data: 49152, used: 17067998, disk: 17092608, relocated: 0, retries: 0, skipped: 0, cleaned:0, log:(0 - 17092608), disk:(0-17092608), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:24 ms elapsed: 24 ms 2021-09-13T16:02:43.208-07:00 [Info] memstats {"Alloc":194116616, "TotalAlloc":173821284864, "Sys":421800184, "Lookups":0, "Mallocs":5624849,"Frees":5609806, "HeapAlloc":194116616, "HeapSys":398360576, "HeapIdle":202080256, "HeapInuse":196280320,"HeapReleased":80478208, "HeapObjects":15043,"GCSys":13348864, "LastGC":1631574163146492293,"PauseTotalNs":6847045933, "PauseNs":[825618, 1724990, 1643911, 1714464, 1227444, 782409, 1181077, 1514285, 1438155, 650244, 1173534, 1317249, 1644743, 1079177, 1398504, 1256298, 2035003, 1370125, 940645, 408650, 1559447, 1619176, 1525971, 1159092, 1020553, 1663369, 1584916, 914836, 1113392, 1775155, 1685434, 1704145, 1034830, 1676688, 1653580, 1876502, 1236685, 1907963, 1229539, 597877, 1182480, 1263543, 1820468, 1302087, 915900, 1647066, 548817, 1247152, 1075476, 581780, 846540, 1312379, 1779803, 1164073, 1486289, 1252976, 1027182, 1827689, 1469612, 1176239, 1673497, 1901149, 2021534, 270619, 1082484, 1662973, 1457966, 1132502, 290466, 3753272, 1197188, 1827763, 1716641, 1770479, 869577, 1931595, 1285199, 669105, 1708664, 1710533, 1872592, 1522514, 1350080, 1676896, 908460, 855494, 1201999, 1025142, 1106108, 2019914, 1200012, 1040964, 1347721, 1719879, 1986260, 1651806, 1151600, 1532447, 2237240, 1483184, 1129836, 1359878, 1648093, 1528018, 1932725, 1002789, 951088, 1732503, 18970366, 1356630, 1507340, 1236578, 984738, 1030940, 1326814, 1336603, 1393591, 477509, 1679922, 964397, 2241494, 1594804, 1637303, 2079541, 666610, 1319772, 1247356, 2321261, 1926693, 744480, 1398173, 1780189, 3577242, 1159736, 2093129, 864894, 1295563, 1557935, 843603, 2644472, 1575042, 1368730, 1290961, 1429680, 574677, 770546, 1340821, 1312854, 1251987, 1715157, 2308416, 1797865, 783146, 984153, 2146066, 3356808, 2454651, 2793531, 1856569, 845393, 1824384, 1541469, 1674462, 3227702, 1390664, 1645213, 2612116, 1766858, 1647779, 1208216, 1261219, 1333867, 1419774, 1406605, 1100975, 1458203, 542903, 1230078, 2784870, 2445847, 1918763, 1294752, 432714, 1640787, 1558564, 1113519, 1847505, 1522943, 1520242, 1317966, 1155809, 1036282, 2653773, 794043, 1563167, 1743006, 835141, 2056460, 1718884, 1134081, 1158645, 1131057, 1043050, 1161098, 823776, 1095380, 3478172, 1520487, 890711, 1434137, 1357649, 1069104, 937036, 1249087, 1506574, 1875911, 1769116, 1711832, 1005613, 1565633, 2551980, 2293004, 991987, 1398538, 853589, 1613817, 851834, 1235639, 1271716, 1920533, 1019240, 1591442, 1541678, 1681558, 1566682, 1084149, 883678, 1153858, 1728704, 1888247, 4904348, 2959378, 1358027, 979924, 1618115, 1192004, 1042297, 1711518, 991843, 1072509, 1845959, 1725734, 1642547, 1623871, 1330682, 1430793], "NumGC":1788} Allocated: 1675123584, active: 1675145216, metadata: 12320256 (n_thp 0), resident: 1704574976, mapped: 1718579200, retained: 1381011456 2021-09-13 16:02:43 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: starting... frag 49, data: 14569335, used: 29073408 log:(0 - 29073408) head:0 2021-09-13 16:02:43 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: completed... frag 99, data: 8149982, used: 2526220258, disk: 2555289600, relocated: 2, retries: 248, skipped: 0, cleaned:29069342, log:(29069342 - 2555289600), disk:(0-2555289600), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:99206 ms elapsed: 99206 ms 2021-09-13 16:02:43 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: starting... frag 49, data: 14959441, used: 29913088 log:(0 - 29913088) head:0 2021-09-13 16:02:43 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: completed... frag 99, data: 16566588, used: 2267751082, disk: 2297663488, relocated: 2, retries: 232, skipped: 0, cleaned:29912406, log:(29912406 - 2297663488), disk:(0-2297663488), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:94508 ms elapsed: 94508 ms 2021-09-13 16:03:01 [ss-workload] instance inse% dele% read% 2021-09-13 16:03:01 [ss-workload] 0 16% 16% 67% 2021-09-13 16:03:01 [ss-workload] 1 16% 16% 67% Mon Sep 13 16:03:12 PDT 2021 RSS = 1579124 2021-09-13T16:03:13.236-07:00 [Info] memstats {"Alloc":159522992, "TotalAlloc":174585679328, "Sys":421800184, "Lookups":0, "Mallocs":5652575,"Frees":5636931, "HeapAlloc":159522992, "HeapSys":398983168, "HeapIdle":237428736, "HeapInuse":161554432,"HeapReleased":192872448, "HeapObjects":15644,"GCSys":13250560, "LastGC":1631574190323358804,"PauseTotalNs":6851462361, "PauseNs":[320881, 917849, 370920, 390927, 444960, 511806, 1106330, 352755], "NumGC":1796} Allocated: 1631411072, active: 1631432704, metadata: 12320256 (n_thp 0), resident: 1661517824, mapped: 1675522048, retained: 1424068608 2021-09-13 16:03:01 [ss-workload] 2 15% 15% 68% 2021-09-13T16:03:43.284-07:00 [Info] memstats {"Alloc":236458712, "TotalAlloc":177437846888, "Sys":421800184, "Lookups":0, "Mallocs":5745090,"Frees":5729965, "HeapAlloc":236458712, "HeapSys":398753792, "HeapIdle":160129024, "HeapInuse":238624768,"HeapReleased":98050048, "HeapObjects":15125,"GCSys":13348864, "LastGC":1631574223284200442,"PauseTotalNs":6885500333, "PauseNs":[536195, 326551, 806691, 358704, 444513, 491333, 359741, 366555, 366246, 425170, 1712114, 1488846, 2635563, 1710547, 2116153, 1742479, 1819352, 1189603, 1799393, 1387133, 1440060, 2403966, 1938686, 2179347, 1991665, 887993, 1113373], "NumGC":1823} Allocated: 1643502464, active: 1643524096, metadata: 12320256 (n_thp 0), resident: 1655832576, mapped: 1669836800, retained: 1429753856 2021-09-13 16:03:01 [ss-workload] 3 15% 15% 68% 2021-09-13 16:03:01 [ss-workload] 4 3% 3% 92% 2021-09-13 16:03:01 [ss-workload] 5 3% 3% 92% 2021-09-13 16:03:01 [ss-workload] 6 3% 3% 92% 2021-09-13 16:03:01 [ss-workload] 7 19% 19% 61% 2021-09-13 16:03:01 [ss-workload] 8 18% 18% 62% 2021-09-13 16:03:01 [ss-workload] 9 19% 19% 61% 2021-09-13 16:03:01 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:03:01 [ss-diag] 0 6000 5575 0 0 227 1 14879396 1 27867692 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:03:01 [ss-diag] 1 6786 6534 0 0 465 2 30479820 1 16520392 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:03:01 [ss-diag] 2 5500 5132 0 0 252 1 16518096 1 24128376 0 1.000000 2.955823 1.000000 0:0:0 2021-09-13 16:03:01 [ss-diag] 3 5144 4696 0 0 259 1 16976932 1 29375192 0 1.000000 2.955823 1.000000 0:0:0 2021-09-13 16:03:01 [ss-diag] 4 26369 25769 0 0 337 2 22089676 2 39406620 65588 1.000000 8.826176 1.000000 0:0:0 2021-09-13 16:03:01 [ss-diag] 5 23800 23196 0 0 279 2 18287892 2 39669484 65588 1.000000 2.955823 1.000000 0:0:0 2021-09-13 16:03:01 [ss-diag] 6 25491 24853 0 0 381 2 24973788 2 41899444 65604 1.000000 2.955823 1.000000 0:0:0 2021-09-13 16:03:01 [ss-diag] 7 11781 11387 0 0 647 2 42409556 1 25831904 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:03:01 [ss-diag] 8 10002 9655 0 0 237 1 15534876 1 22751508 0 1.000000 2.955823 1.000000 0:0:0 2021-09-13 16:03:54 [gsi-test] [] QuotaTuner: 5368709120 -> 4831838208 2021-09-13 16:03:01 [ss-diag] 9 9805 9525 0 0 250 1 16387000 1 18356088 0 1.000000 2.955823 1.000000 0:0:0 Mon Sep 13 16:04:12 PDT 2021 RSS = 1581008 2021-09-13T16:04:13.517-07:00 [Info] memstats {"Alloc":195866232, "TotalAlloc":232173779704, "Sys":421800184, "Lookups":0, "Mallocs":7511946,"Frees":7495125, "HeapAlloc":195866232, "HeapSys":398491648, "HeapIdle":200597504, "HeapInuse":197894144,"HeapReleased":75948032, "HeapObjects":16821,"GCSys":13250560, "LastGC":1631574253479568054,"PauseTotalNs":7641044532, "PauseNs":[1484450, 721309, 2092838, 1416231, 757985, 1250883, 1001146, 1113461, 1014244, 1306646, 787169, 1487654, 1684213, 1642329, 1680975, 944227, 1442390, 1764165, 1769943, 1170717, 1775366, 1432295, 990584, 1485783, 2658484, 893179, 1966665, 823789, 1352939, 847206, 1354102, 2222831, 1129777, 1452140, 1523854, 1284886, 529848, 1081183, 869083, 1256130, 1346969, 2004405, 893441, 808043, 1323117, 1731107, 1580808, 1088465, 1732335, 779581, 1029608, 1574127, 2171802, 839834, 1339435, 1575924, 1271786, 1632880, 1324474, 1844006, 1784210, 1745929, 1100346, 763219, 1333187, 1188337, 427070, 1311174, 1491218, 1721030, 1164458, 1825961, 1188565, 1297927, 1565256, 953874, 2168022, 1392285, 1118926, 1321112, 1917071, 904775, 496699, 909623, 1482216, 576631, 1592639, 1486468, 1962469, 1555044, 488639, 710097, 1348178, 1668621, 1158316, 1166387, 1037450, 1069084, 2385453, 2280629, 1057096, 1417964, 1124906, 1343400, 1337217, 1177454, 1688985, 1277768, 1060222, 1430561, 1378288, 1351183, 1571426, 1446406, 1482148, 423091, 1103787, 1442865, 2269505, 1248823, 1757201, 1062094, 1015763, 1687222, 791722, 1719048, 2329715, 1132557, 2065487, 1106752, 1391728, 1464799, 1309171, 1691070, 1188387, 1861024, 1422179, 455244, 1689885, 1598032, 810874, 1329198, 1287102, 1649934, 1174716, 1338597, 3369685, 1493610, 1250215, 784006, 669041, 2282220, 1434821, 1772107, 1599264, 1173219, 1105712, 737648, 1032732, 1233268, 1517301, 1193616, 1326045, 1916206, 1441897, 1447196, 816808, 1076390, 2124446, 1545326, 1298528, 826046, 831964, 1410942, 1249160, 1354746, 1412521, 1283813, 1553531, 1271132, 809035, 901262, 700787, 1950922, 1929010, 1308677, 951049, 1701053, 563521, 1366996, 997158, 924869, 1209393, 796255, 1688934, 1348193, 1781429, 1036192, 1135075, 1323992, 1679728, 1665399, 1222955, 494748, 2441872, 1583097, 1157424, 1005004, 1460631, 1025115, 1409744, 958065, 1600223, 1407289, 1025236, 1600834, 1418385, 1222552, 1261708, 1138281, 2377642, 1433769, 1131693, 881892, 1278717, 1007930, 1198052, 1957561, 804264, 1249925, 1035387, 1041067, 1104607, 1283656, 909218, 1906451, 1546602, 1873395, 1631195, 1799665, 1499584, 1681392, 1213076, 902261, 1431432, 623503, 1374034, 940015, 1091542, 1406940, 1764622, 2383453, 602544, 975478, 1084165, 1125247], "NumGC":2299} Allocated: 1665915776, active: 1665937408, metadata: 12320256 (n_thp 0), resident: 1677099008, mapped: 1691103232, retained: 1408487424 2021-09-13T16:04:43.716-07:00 [Info] memstats {"Alloc":126890792, "TotalAlloc":285943044632, "Sys":421865720, "Lookups":0, "Mallocs":9254601,"Frees":9240016, "HeapAlloc":126890792, "HeapSys":398426112, "HeapIdle":269451264, "HeapInuse":128974848,"HeapReleased":102137856, "HeapObjects":14585,"GCSys":13316096, "LastGC":1631574283598524425,"PauseTotalNs":8282907584, "PauseNs":[1099525, 1214129, 1535839, 2530367, 1365343, 840927, 544594, 955789, 1280946, 818067, 1178094, 1926711, 1541412, 1512064, 1528877, 1341537, 1321770, 1278852, 1071751, 1550728, 812150, 3987433, 2273875, 1477355, 811049, 1109838, 1259786, 785237, 1452513, 676665, 865596, 1980365, 818858, 627086, 2707220, 1804150, 397289, 1802333, 1635440, 1187509, 1622792, 1761511, 1641992, 862911, 1772835, 966867, 987200, 2149013, 864551, 1451224, 930267, 844828, 1909783, 1252308, 1091931, 1120836, 941736, 1471714, 1565356, 1407032, 1207151, 1124759, 1035580, 1163382, 833068, 323726, 1541755, 886041, 1389418, 1045230, 1962250, 1414646, 1641901, 1731403, 1617748, 1134342, 2033907, 1088652, 1387636, 1611586, 1078788, 1519138, 1296913, 1328564, 895077, 1412609, 1384550, 1674330, 1793736, 1319560, 862522, 1194555, 1606583, 1256726, 1807499, 1563683, 1337601, 884585, 825656, 1783207, 548530, 1127338, 2095480, 691168, 1340039, 953475, 1505670, 1332134, 1466069, 1339075, 1763609, 1199118, 837014, 641257, 912731, 1172824, 1351172, 1401291, 1060807, 2152556, 1323860, 2666954, 1782937, 971901, 1020200, 1554547, 1953007, 1521553, 1411221, 688977, 723734, 930684, 1406220, 1475643, 1151790, 1616337, 1493228, 1832705, 1115337, 1623599, 1750717, 1349815, 1663290, 1479607, 2365871, 1666530, 1904250, 1317013, 1350654, 1251642, 1756503, 1582557, 585457, 2713069, 1081568, 1055998, 805419, 1681527, 1102551, 1511628, 843626, 1543341, 1261495, 1777174, 1347956, 1218963, 1867551, 1121540, 1500360, 2067228, 1101797, 1757203, 1570916, 839231, 1108224, 1520558, 1746669, 1275784, 759426, 1369975, 1915401, 1379417, 1926000, 1545790, 958468, 1299098, 1611265, 1559183, 1169531, 1967885, 1295522, 2044921, 1471056, 1800833, 1107370, 1676563, 1616066, 2567675, 1039335, 1086173, 1584661, 1050041, 1100549, 1937938, 958301, 1416521, 1216032, 986852, 1437616, 1111146, 1370646, 1251825, 1514871, 745540, 2182534, 1205023, 1438329, 1685380, 1603088, 1291067, 1459537, 1172042, 1463408, 1699294, 1052925, 1522662, 1230230, 1376470, 1286130, 1588739, 1088766, 1366502, 1053173, 1690455, 1646643, 937450, 1559379, 677806, 1792235, 1562131, 1733317, 483188, 2173368, 2160658, 1222661, 1495142, 1943866, 2326831, 1313077, 1369572, 1139280, 1333855, 1301047, 1185856, 1026346, 1311505], "NumGC":2760} Allocated: 1653824384, active: 1653846016, metadata: 12320256 (n_thp 0), resident: 1706131456, mapped: 1720135680, retained: 1379454976 2021-09-13 16:04:54 [ss-workload] instance inse% dele% read% 2021-09-13 16:04:54 [ss-workload] 0 15% 15% 68% 2021-09-13 16:04:54 [ss-workload] 1 15% 15% 68% Mon Sep 13 16:05:12 PDT 2021 RSS = 1619404 2021-09-13T16:05:13.751-07:00 [Info] memstats {"Alloc":193324608, "TotalAlloc":295063112928, "Sys":421865720, "Lookups":0, "Mallocs":9554874,"Frees":9540243, "HeapAlloc":193324608, "HeapSys":398622720, "HeapIdle":203161600, "HeapInuse":195461120,"HeapReleased":193249280, "HeapObjects":14631,"GCSys":13381632, "LastGC":1631574313747805113,"PauseTotalNs":8713075103, "PauseNs":[68702749, 403656, 1007858, 129064470, 558077, 13364984, 422034, 521207, 144983899, 1130475, 823090, 436683, 437175, 485957, 616968, 851737, 1007842, 885873, 905208, 476418, 771528, 533802, 5338212, 1077879, 1140949, 1035300, 953972, 649825, 550473, 1061511, 369246, 579309, 452019, 691000, 510218, 453523, 929792, 416807, 489355, 928233, 613376, 542827, 1007023, 951903, 748970, 669533, 687631, 785236, 997417, 693638, 675238, 561493, 1135064, 946582, 1119500, 1016623, 341323, 463983, 1064744, 1691415, 870822, 859207, 396669, 384767, 336497, 446862, 404255, 1072396, 940903, 612290, 965647, 738703, 560848, 1049815, 823830, 365115, 1302030, 424778, 1336305, 434090, 1221349, 433846, 434564, 513892, 1409524, 393416, 585143, 373784, 1105435, 416509, 1124084, 406674, 503695, 1578304, 534542, 454748, 1619349], "NumGC":2857} Allocated: 1687411584, active: 1687433216, metadata: 12320256 (n_thp 0), resident: 1699086336, mapped: 1713090560, retained: 1386500096 2021-09-13 16:05:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 195, "num_active_sctxs": 86, "num_free_sctxs": 109, "num_trim_sctxs": 116, "num_wctxs": 239, "num_free_wctxs": 183, "num_active_wctxs": 56, "num_trim_wctxs": 36, "num_readers": 102, "num_writers": 115, "cpu": 8.66000, "total_core": 16, "threads": 45, "assigned_quota": 4831838208, "current_quota": 4831838208, "memory_resident": 0, "memory_jemalloc": 1695514624, "memory_jemalloc_allocated": 1647631232, "memory_in_use": 1695514624, "memory_stats_size_total": 1511692998, "memory_stats_size_global": 262460428, "memory_stats_size_page": 262264672, "memory_stats_size_index": 196780, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 50608, "reclaim_pending": 0, "buf_memused": 1249231546, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 13, "items_count": 2079, "total_records": 3997, "total_ops": 94089, "compacts": 465, "purges": 0, "lss_data_size": 44166511, "lss_used_space": 67272079, "lss_fragmentation": 34, "lss_num_reads": 1046, "lss_read_bs": 6479470097, "lss_blk_read_bs": 6491815936, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 6547496960, "bytes_incoming": 6167345772, "write_amp_avg": 0.73, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 0 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 3997, "resident_ratio": 100.00000, "cleaner_num_reads": 1046, "cleaner_read_bs": 6479470097, "cleaner_blk_read_bs": 6491815936, "recovery_num_lssctx": 5, "recovery_num_sctxs": 25, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 5, "recovery_num_trim_sctxs": 5, "recovery_buf_memused": 1048576, "recovery_data_size": 344256, "recovery_used_space": 26520830, "recovery_lss_blk_read_bs": 30371840, "recovery_lss_read_bs": 19418864, "recovery_lss__num_reads": 320, "recovery_bytes_written": 45322240, "recovery_lss_frag": 98, "recovery_cleaner_num_reads": 320, "recovery_cleaner_read_bs": 19418864, "recovery_cleaner_blk_read_bs": 30371840, "mvcc_purge_ratio": 1.92256 }2021-09-13 16:04:54 [ss-workload] 2 15% 15% 68% 2021-09-13 16:04:54 [ss-workload] 3 15% 15% 68% 2021-09-13 16:04:54 [ss-workload] 4 3% 3% 92% 2021-09-13 16:04:54 [ss-workload] 5 3% 3% 92% 2021-09-13 16:04:54 [ss-workload] 6 3% 3% 92% 2021-09-13 16:05:43 [gsi-test] [] Stop and verify... 2021-09-13 16:05:43 [gsi-test] [] { "memory_quota": 4831838208, "count": 4263, "compacts": 21, "purges": 0, "splits": 0, "merges": 0, "inserts": 4263, "deletes": 0, "compact_conflicts": 63, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 1246, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 16849292, "memory_size_index": 0, "allocated": 610332872, "freed": 593483580, "reclaimed": 593483580, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 0, "freed_index": 0, "reclaimed_index": 0, "num_pages": 1, "items_count": 207, "total_records": 257, "num_rec_allocs": 9308, "num_rec_frees": 9051, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 279431124, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 8192, "est_recovery_size": 8212, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 12559, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 8296, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 1.24155, "currSn": 315, "gcSn": 312, "gcSnIntervals": "[0 313]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 2, "num_free_wctxs": 22, "num_trim_wctxs": 4, "num_readers": 9, "num_writers": 12, "page_bytes": 14092820, "page_cnt": 1, "page_itemcnt": 215, "avg_item_size": 65548, "avg_page_size": 14092820, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 15800044, "page_bytes_marshalled":0, "page_bytes_compressed":0, "compression_ratio": 0.00000, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 1736, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 458836, "lss_data_size": 16384, "lss_used_space": 8192, "lss_disk_size": 8192, "lss_fragmentation": 0, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "bytes_written": 8192, "bytes_incoming": 279431124, "write_amp": 0.00, "write_amp_avg": 0.00, "lss_gc_num_reads": 0, "lss_gc_reads_bs": 0, "lss_blk_gc_reads_bs": 0, "lss_gc_status": "", "lss_head_offset": 0, "lss_tail_offset": 4096, "num_sctxs": 26, "num_free_sctxs": 18, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:05:43 [gsi-test] [] { "memory_quota": 4831838208, "count": 4428, "compacts": 22, "purges": 0, "splits": 0, "merges": 0, "inserts": 4428, "deletes": 0, "compact_conflicts": 54, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 1147, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 15340352, "memory_size_index": 0, "allocated": 642263980, "freed": 626923628, "reclaimed": 626923628, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 0, "freed_index": 0, "reclaimed_index": 0, "num_pages": 1, "items_count": 208, "total_records": 234, "num_rec_allocs": 9795, "num_rec_frees": 9561, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 290246544, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 8192, "est_recovery_size": 8212, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 12996, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 8568, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 1.12500, "currSn": 315, "gcSn": 312, "gcSnIntervals": "[0 313]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 2, "num_free_wctxs": 25, "num_trim_wctxs": 4, "num_readers": 11, "num_writers": 13, "page_bytes": 28644476, "page_cnt": 2, "page_itemcnt": 437, "avg_item_size": 65548, "avg_page_size": 14322238, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 15209784, "page_bytes_marshalled":0, "page_bytes_compressed":0, "compression_ratio": 0.00000, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 296, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 524384, "lss_data_size": 16384, "lss_used_space": 8192, "lss_disk_size": 8192, "lss_fragmentation": 0, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "bytes_written": 8192, "bytes_incoming": 290246544, "write_amp": 0.00, "write_amp_avg": 0.00, "lss_gc_num_reads": 0, "lss_gc_reads_bs": 0, "lss_blk_gc_reads_bs": 0, "lss_gc_status": "", "lss_head_offset": 0, "lss_tail_offset": 4096, "num_sctxs": 31, "num_free_sctxs": 22, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13T16:05:43.802-07:00 [Info] memstats {"Alloc":248550496, "TotalAlloc":306129046504, "Sys":421865720, "Lookups":0, "Mallocs":9915962,"Frees":9899904, "HeapAlloc":248550496, "HeapSys":398589952, "HeapIdle":147914752, "HeapInuse":250675200,"HeapReleased":90644480, "HeapObjects":16058,"GCSys":13381632, "LastGC":1631574343800645593,"PauseTotalNs":8823229222, "PauseNs":[858203, 385868, 1073606, 1269005, 455443, 374782, 1653146, 1557614, 1157950, 1413873, 547267, 540181, 1461844, 778863, 833257, 564371, 406008, 379638, 440736, 1120545, 1600487, 1074979, 1539421, 595672, 1241932, 401670, 433313, 420144, 462266, 478333, 413778, 1372207, 561206, 401131, 837187, 619963, 606522, 1622706, 661311, 402227, 1246787, 956851, 541186, 1060324, 345462, 469209, 1110475, 1086078, 1745671, 405127, 549560, 433569, 552797, 409108, 509876, 2478815, 1395926, 1874661, 908583, 422516, 399829, 1107142, 708122, 433996, 716252, 324074, 1015034, 425463, 555140, 649951, 905707, 1129190, 380912, 917378, 851808, 453533, 1173261, 853042, 840528, 895473, 1190781, 416912, 1117340, 1140067, 427226, 553810, 459249, 742451, 788496, 1507552, 1392290, 2337125, 1920363, 896038, 3029199, 2156360, 1569870, 1227055, 2023971, 1352091, 1451404, 2343227, 2093246, 856923, 933444, 2260747, 1185710, 1548844, 1444398, 1974749, 1556110], "NumGC":2968} Allocated: 1605081984, active: 1605103616, metadata: 12320256 (n_thp 0), resident: 1680048128, mapped: 1694052352, retained: 1405538304 2021-09-13 16:04:54 [ss-workload] 7 18% 18% 62% 2021-09-13 16:05:44 [gsi-test] [] { "memory_quota": 4831838208, "count": 3470, "compacts": 17, "purges": 0, "splits": 0, "merges": 0, "inserts": 3470, "deletes": 0, "compact_conflicts": 45, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 994, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 20193000, "memory_size_index": 0, "allocated": 504958780, "freed": 484765780, "reclaimed": 484765780, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 0, "freed_index": 0, "reclaimed_index": 0, "num_pages": 1, "items_count": 208, "total_records": 308, "num_rec_allocs": 7701, "num_rec_frees": 7393, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 227451560, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 819200, "est_recovery_size": 24708, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 10345, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 6875, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 1.48077, "currSn": 200, "gcSn": 197, "gcSnIntervals": "[0 198]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 2, "num_free_wctxs": 16, "num_trim_wctxs": 7, "num_readers": 9, "num_writers": 7, "page_bytes": 16714740, "page_cnt": 1, "page_itemcnt": 255, "avg_item_size": 65548, "avg_page_size": 16714740, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 18487712, "page_bytes_marshalled":0, "page_bytes_compressed":0, "compression_ratio": 0.00000, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 2176, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 982149520, "lss_data_size": 98304, "lss_used_space": 54742236, "lss_disk_size": 76562432, "lss_fragmentation": 99, "lss_num_reads": 808, "lss_read_bs": 4182266598, "lss_blk_read_bs": 4195196928, "bytes_written": 4237312000, "bytes_incoming": 3747116968, "write_amp": 3.81, "write_amp_avg": 1.12, "lss_gc_num_reads": 808, "lss_gc_reads_bs": 4182266598, "lss_blk_gc_reads_bs": 4195196928, "lss_gc_status": "frag 74, data: 7789459, used: 30025857 log:(4163762047 - 4193787904) head:2581323824", "lss_head_offset": 4163762047, "lss_tail_offset": 4207534080, "num_sctxs": 81, "num_free_sctxs": 72, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:05:44 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper stopped 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:05:44 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : All daemons stopped 2021-09-13 16:05:44 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper stopped 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:05:44 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : All daemons stopped 2021-09-13 16:05:44 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper stopped 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:05:44 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:05:45 [gsi-test] [] { "memory_quota": 4831838208, "count": 3250, "compacts": 16, "purges": 0, "splits": 0, "merges": 0, "inserts": 3250, "deletes": 0, "compact_conflicts": 64, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 1135, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 16390144, "memory_size_index": 0, "allocated": 470665896, "freed": 454275752, "reclaimed": 454275752, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 0, "freed_index": 0, "reclaimed_index": 0, "num_pages": 1, "items_count": 208, "total_records": 250, "num_rec_allocs": 7178, "num_rec_frees": 6928, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 213031000, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 819200, "est_recovery_size": 24708, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 9661, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 6411, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 1.20192, "currSn": 205, "gcSn": 203, "gcSnIntervals": "[0 204]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 2, "num_free_wctxs": 21, "num_trim_wctxs": 3, "num_readers": 10, "num_writers": 10, "page_bytes": 32577356, "page_cnt": 2, "page_itemcnt": 497, "avg_item_size": 65548, "avg_page_size": 16288678, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 31079776, "page_bytes_marshalled":0, "page_bytes_compressed":0, "compression_ratio": 0.00000, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 1416, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 982149520, "lss_data_size": 98304, "lss_used_space": 63642844, "lss_disk_size": 85479424, "lss_fragmentation": 99, "lss_num_reads": 817, "lss_read_bs": 4204526979, "lss_blk_read_bs": 4216434688, "bytes_written": 4246228992, "bytes_incoming": 3769927672, "write_amp": 3.81, "write_amp_avg": 1.12, "lss_gc_num_reads": 817, "lss_gc_reads_bs": 4204526979, "lss_blk_gc_reads_bs": 4216434688, "lss_gc_status": "frag 74, data: 7789459, used: 30025857 log:(4163762047 - 4193787904) head:2581323824", "lss_head_offset": 4163762047, "lss_tail_offset": 4216434688, "num_sctxs": 81, "num_free_sctxs": 70, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:05:45 [gsi-test] [] { "memory_quota": 4831838208, "count": 24300, "compacts": 121, "purges": 0, "splits": 1, "merges": 0, "inserts": 24300, "deletes": 0, "compact_conflicts": 79, "split_conflicts": 2, "merge_conflicts": 0, "insert_conflicts": 920, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 34816228, "memory_size_index": 65588, "allocated": 3010722528, "freed": 2975906300, "reclaimed": 2975906300, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 65588, "freed_index": 0, "reclaimed_index": 0, "num_pages": 2, "items_count": 208, "total_records": 530, "num_rec_allocs": 45854, "num_rec_frees": 45324, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1592816400, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 819200, "est_recovery_size": 352606, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 280114, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 255814, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 2.54808, "currSn": 4703, "gcSn": 4673, "gcSnIntervals": "[0 4674]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 26, "num_trim_wctxs": 1, "num_readers": 16, "num_writers": 8, "page_bytes": 26415844, "page_cnt": 3, "page_itemcnt": 403, "avg_item_size": 65548, "avg_page_size": 8805281, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 39081528, "page_bytes_marshalled":68629030, "page_bytes_compressed":37360755, "compression_ratio": 1.83693, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 7472, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 265508874, "lss_data_size": 16384, "lss_used_space": 26515472, "lss_disk_size": 2324176896, "lss_fragmentation": 99, "lss_num_reads": 243, "lss_read_bs": 2297203637, "lss_blk_read_bs": 2297667584, "bytes_written": 2324176896, "bytes_incoming": 1592816400, "write_amp": 0.00, "write_amp_avg": 1.45, "lss_gc_num_reads": 243, "lss_gc_reads_bs": 2297203637, "lss_blk_gc_reads_bs": 2297667584, "lss_gc_status": "frag 0, data: 16566588, used: 10713104, disk: 2308374528, relocated: 3, retries: 232, skipped: 0, cleaned:2267749018, log:(2297661424 - 2308374528), disk:(0-2308374528), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:1922 ms elapsed: 1922 ms", "lss_head_offset": 2297661424, "lss_tail_offset": 2308374528, "num_sctxs": 34, "num_free_sctxs": 26, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:05:45 [gsi-test] [] { "memory_quota": 4831838208, "count": 20192, "compacts": 101, "purges": 0, "splits": 1, "merges": 0, "inserts": 20192, "deletes": 0, "compact_conflicts": 115, "split_conflicts": 2, "merge_conflicts": 0, "insert_conflicts": 869, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 27077900, "memory_size_index": 65588, "allocated": 2476636628, "freed": 2449558728, "reclaimed": 2449558728, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 65588, "freed_index": 0, "reclaimed_index": 0, "num_pages": 2, "items_count": 208, "total_records": 412, "num_rec_allocs": 37722, "num_rec_frees": 37310, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1323545216, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 819200, "est_recovery_size": 352738, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 234394, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 214202, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 1.98077, "currSn": 2969, "gcSn": 2945, "gcSnIntervals": "[0 2946]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 31, "num_trim_wctxs": 0, "num_readers": 16, "num_writers": 13, "page_bytes": 10159940, "page_cnt": 1, "page_itemcnt": 155, "avg_item_size": 65548, "avg_page_size": 10159940, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 39473136, "page_bytes_marshalled":140470890, "page_bytes_compressed":76462030, "compression_ratio": 1.83713, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 4072, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 982149520, "lss_data_size": 98304, "lss_used_space": 73596124, "lss_disk_size": 95432704, "lss_fragmentation": 99, "lss_num_reads": 817, "lss_read_bs": 4204526979, "lss_blk_read_bs": 4216434688, "bytes_written": 4256182272, "bytes_incoming": 3773205072, "write_amp": 3.81, "write_amp_avg": 1.12, "lss_gc_num_reads": 817, "lss_gc_reads_bs": 4204526979, "lss_blk_gc_reads_bs": 4216434688, "lss_gc_status": "frag 74, data: 7789459, used: 30025857 log:(4163762047 - 4193787904) head:2581323824", "lss_head_offset": 4163762047, "lss_tail_offset": 4226387968, "num_sctxs": 81, "num_free_sctxs": 72, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:05:45 [gsi-test] [] { "memory_quota": 4831838208, "count": 19166, "compacts": 95, "purges": 0, "splits": 1, "merges": 0, "inserts": 19166, "deletes": 0, "compact_conflicts": 136, "split_conflicts": 14, "merge_conflicts": 0, "insert_conflicts": 1004, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 39932604, "memory_size_index": 65604, "allocated": 2582935244, "freed": 2543002640, "reclaimed": 2543002640, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 65604, "freed_index": 0, "reclaimed_index": 0, "num_pages": 2, "items_count": 208, "total_records": 608, "num_rec_allocs": 39354, "num_rec_frees": 38746, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1256292968, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 819200, "est_recovery_size": 287132, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 218797, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 199631, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 2.92308, "currSn": 2693, "gcSn": 2667, "gcSnIntervals": "[0 2668]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 2, "num_free_wctxs": 30, "num_trim_wctxs": 3, "num_readers": 16, "num_writers": 15, "page_bytes": 23335088, "page_cnt": 2, "page_itemcnt": 356, "avg_item_size": 65548, "avg_page_size": 11667544, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 40914904, "page_bytes_marshalled":101141464, "page_bytes_compressed":55049137, "compression_ratio": 1.83729, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 11232, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 982149520, "lss_data_size": 98304, "lss_used_space": 73596124, "lss_disk_size": 95432704, "lss_fragmentation": 99, "lss_num_reads": 817, "lss_read_bs": 4204526979, "lss_blk_read_bs": 4216434688, "bytes_written": 4256182272, "bytes_incoming": 3773205072, "write_amp": 3.81, "write_amp_avg": 1.12, "lss_gc_num_reads": 817, "lss_gc_reads_bs": 4204526979, "lss_blk_gc_reads_bs": 4216434688, "lss_gc_status": "frag 74, data: 7789459, used: 30025857 log:(4163762047 - 4193787904) head:2581323824", "lss_head_offset": 4163762047, "lss_tail_offset": 4226387968, "num_sctxs": 81, "num_free_sctxs": 72, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:04:54 [ss-workload] 8 18% 18% 62% 2021-09-13 16:04:54 [ss-workload] 9 19% 19% 61% 2021-09-13 16:04:54 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:05:47 [gsi-test] [] { "memory_quota": 4831838208, "count": 7132, "compacts": 35, "purges": 0, "splits": 0, "merges": 0, "inserts": 7132, "deletes": 0, "compact_conflicts": 137, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 2328, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 20063296, "memory_size_index": 0, "allocated": 1074693056, "freed": 1054629760, "reclaimed": 1054629760, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 0, "freed_index": 0, "reclaimed_index": 0, "num_pages": 1, "items_count": 208, "total_records": 306, "num_rec_allocs": 16390, "num_rec_frees": 16084, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 467488336, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 8192, "est_recovery_size": 8212, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 18680, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 11548, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 1.47115, "currSn": 374, "gcSn": 370, "gcSnIntervals": "[0 371]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 2, "num_free_wctxs": 18, "num_trim_wctxs": 4, "num_readers": 5, "num_writers": 14, "page_bytes": 28644476, "page_cnt": 2, "page_itemcnt": 437, "avg_item_size": 65548, "avg_page_size": 14322238, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 22030680, "page_bytes_marshalled":0, "page_bytes_compressed":0, "compression_ratio": 0.00000, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 3936, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 852124, "lss_data_size": 16384, "lss_used_space": 8192, "lss_disk_size": 8192, "lss_fragmentation": 0, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "bytes_written": 8192, "bytes_incoming": 467488336, "write_amp": 0.00, "write_amp_avg": 0.00, "lss_gc_num_reads": 0, "lss_gc_reads_bs": 0, "lss_blk_gc_reads_bs": 0, "lss_gc_status": "", "lss_head_offset": 0, "lss_tail_offset": 4096, "num_sctxs": 23, "num_free_sctxs": 13, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:05:47 [gsi-test] [] { "memory_quota": 4831838208, "count": 5784, "compacts": 28, "purges": 0, "splits": 0, "merges": 0, "inserts": 5784, "deletes": 0, "compact_conflicts": 118, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 2270, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 31078592, "memory_size_index": 0, "allocated": 877719304, "freed": 846640712, "reclaimed": 846640712, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 0, "freed_index": 0, "reclaimed_index": 0, "num_pages": 1, "items_count": 208, "total_records": 474, "num_rec_allocs": 13386, "num_rec_frees": 12912, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 379129632, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 819200, "est_recovery_size": 24708, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 15338, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 9554, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 2.27885, "currSn": 229, "gcSn": 226, "gcSnIntervals": "[0 227]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 2, "num_free_wctxs": 17, "num_trim_wctxs": 5, "num_readers": 5, "num_writers": 11, "page_bytes": 20844264, "page_cnt": 1, "page_itemcnt": 318, "avg_item_size": 65548, "avg_page_size": 20844264, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 24257440, "page_bytes_marshalled":0, "page_bytes_compressed":0, "compression_ratio": 0.00000, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 6296, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 982149520, "lss_data_size": 98304, "lss_used_space": 121867484, "lss_disk_size": 143704064, "lss_fragmentation": 99, "lss_num_reads": 817, "lss_read_bs": 4204526979, "lss_blk_read_bs": 4216434688, "bytes_written": 4304453632, "bytes_incoming": 3812861612, "write_amp": 3.81, "write_amp_avg": 1.12, "lss_gc_num_reads": 817, "lss_gc_reads_bs": 4204526979, "lss_blk_gc_reads_bs": 4216434688, "lss_gc_status": "frag 74, data: 7789459, used: 30025857 log:(4163762047 - 4193787904) head:2581323824", "lss_head_offset": 4163762047, "lss_tail_offset": 4274659328, "num_sctxs": 81, "num_free_sctxs": 58, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:04:54 [ss-diag] 0 9354 9051 0 0 215 1 14092820 1 19866340 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:05:47 [gsi-test] [] { "memory_quota": 4831838208, "count": 5742, "compacts": 28, "purges": 0, "splits": 0, "merges": 0, "inserts": 5742, "deletes": 0, "compact_conflicts": 138, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 2190, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 23866088, "memory_size_index": 0, "allocated": 858444496, "freed": 834578408, "reclaimed": 834578408, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 0, "freed_index": 0, "reclaimed_index": 0, "num_pages": 1, "items_count": 208, "total_records": 364, "num_rec_allocs": 13092, "num_rec_frees": 12728, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 376376616, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 819200, "est_recovery_size": 24708, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 15135, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 9393, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 1.75000, "currSn": 228, "gcSn": 225, "gcSnIntervals": "[0 226]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 2, "num_free_wctxs": 18, "num_trim_wctxs": 5, "num_readers": 5, "num_writers": 12, "page_bytes": 16845836, "page_cnt": 1, "page_itemcnt": 257, "avg_item_size": 65548, "avg_page_size": 16845836, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 23865864, "page_bytes_marshalled":0, "page_bytes_compressed":0, "compression_ratio": 0.00000, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 4616, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 982149520, "lss_data_size": 98304, "lss_used_space": 121867484, "lss_disk_size": 143704064, "lss_fragmentation": 99, "lss_num_reads": 817, "lss_read_bs": 4204526979, "lss_blk_read_bs": 4216434688, "bytes_written": 4304453632, "bytes_incoming": 3817122232, "write_amp": 3.81, "write_amp_avg": 1.12, "lss_gc_num_reads": 817, "lss_gc_reads_bs": 4204526979, "lss_blk_gc_reads_bs": 4216434688, "lss_gc_status": "frag 74, data: 7789459, used: 30025857 log:(4163762047 - 4193787904) head:2581323824", "lss_head_offset": 4163762047, "lss_tail_offset": 4274659328, "num_sctxs": 81, "num_free_sctxs": 72, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:04:54 [ss-diag] 1 9882 9561 0 0 228 1 14944944 1 21046508 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:04:54 [ss-diag] 2 7767 7393 0 0 255 1 16714740 1 24521808 0 1.000000 3.805807 1.000000 0:0:0 2021-09-13 16:04:54 [ss-diag] 3 7220 6928 0 0 497 2 32577356 1 19144840 0 1.000000 3.805807 1.000000 0:0:0 2021-09-13 16:04:54 [ss-diag] 4 46660 46072 0 0 403 3 26415844 2 38619884 65588 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:04:54 [ss-diag] 5 38477 38025 0 0 155 1 10159940 2 29700140 65588 1.000000 3.805807 1.000000 0:0:0 2021-09-13 16:04:54 [ss-diag] 6 40024 39478 0 0 356 2 23335088 2 35866468 65604 1.000000 3.805807 1.000000 0:0:0 2021-09-13 16:04:54 [ss-diag] 7 16454 16084 0 0 437 2 28644476 1 24260928 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:04:54 [ss-diag] 8 13803 13431 0 0 318 1 20844264 1 24389608 0 1.000000 3.805807 1.000000 0:0:0 2021-09-13 16:04:54 [ss-diag] 9 13154 12728 0 0 257 1 16845836 1 27932544 0 1.000000 3.805807 1.000000 0:0:0 Mon Sep 13 16:06:12 PDT 2021 RSS = 1613628 2021-09-13T16:06:14.072-07:00 [Info] memstats {"Alloc":181994536, "TotalAlloc":358685714904, "Sys":421865720, "Lookups":0, "Mallocs":11614129,"Frees":11597664, "HeapAlloc":181994536, "HeapSys":398229504, "HeapIdle":214220800, "HeapInuse":184008704,"HeapReleased":101269504, "HeapObjects":16465,"GCSys":13479936, "LastGC":1631574374040801425,"PauseTotalNs":9602154388, "PauseNs":[1563341, 2430101, 724362, 1442286, 754897, 1690112, 1342269, 1360282, 985041, 1678183, 2185324, 1020688, 1665007, 1203416, 1631647, 1685751, 886271, 1426427, 1004499, 1966483, 1009777, 961538, 914927, 1298467, 1531141, 1860700, 1475923, 831981, 1096646, 967344, 1206116, 1870078, 1256432, 1717688, 954838, 1450854, 937342, 1133323, 1399029, 1533832, 1566534, 939096, 1378058, 1205102, 1838038, 1490303, 1014623, 1620545, 1463760, 1391591, 804378, 1288960, 1028981, 1550695, 906487, 1666998, 1274814, 2278937, 1843897, 1050883, 1383282, 1678802, 1899258, 1480559, 1635892, 1857078, 1491504, 776851, 1740980, 824138, 1409869, 1509578, 1926707, 1338877, 934445, 1972341, 1688582, 2316792, 1076822, 946133, 1861707, 1310753, 2154376, 1377207, 1606404, 1275428, 1862218, 2781292, 1580216, 1087914, 762263, 1122375, 1366661, 1126798, 1092564, 691813, 1100477, 1109688, 1377039, 1000378, 1148496, 1151684, 969027, 995635, 1266236, 1549848, 733978, 1463318, 1890289, 1634019, 1675959, 1319469, 1037455, 1126755, 1367301, 1755852, 804093, 1378148, 1716096, 1612605, 877857, 1848462, 781033, 1549751, 1472227, 1789778, 1201995, 2035949, 1569263, 1533276, 1240749, 1299724, 1337722, 1175946, 1224665, 1934099, 1517465, 1836266, 1317526, 1168809, 2112991, 1627594, 966868, 1296570, 850944, 1774811, 1770709, 324610, 1202957, 1365012, 731254, 1544642, 1517370, 1835472, 1640220, 1993060, 564982, 1571325, 1575713, 760372, 780555, 1169933, 1180782, 1157353, 647294, 1130435, 831432, 1974945, 1225686, 1333749, 1559207, 1087402, 1409991, 1281638, 1185267, 1239387, 1696546, 1039983, 1453028, 1508940, 1061870, 1683422, 1503771, 1100578, 1700179, 885774, 878531, 956539, 1332772, 1669818, 2094212, 1305322, 1498089, 2045368, 977027, 1458019, 1850702, 995953, 1703368, 1389641, 3208019, 959915, 2022033, 1146996, 1757640, 645821, 1719401, 1414385, 1982206, 1205540, 938339, 745073, 1562285, 1840695, 1547861, 1251721, 1927165, 1345225, 1210176, 1125451, 1001491, 1472475, 1226133, 1624155, 2939362, 940742, 4588813, 1474766, 1563328, 1626329, 1475164, 1470670, 820162, 1318576, 881262, 1112160, 599071, 798703, 1717787, 1910374, 2264485, 1431031, 978684, 1373428, 1887404, 1313946, 1388103, 982941, 901765, 1564731, 1246484, 1250700, 1683532, 1766633, 1733008, 740210], "NumGC":3422} Allocated: 1647025024, active: 1647046656, metadata: 12320256 (n_thp 0), resident: 1701707776, mapped: 1715712000, retained: 1383878656 2021-09-13T16:06:44.279-07:00 [Info] memstats {"Alloc":191472672, "TotalAlloc":414475674168, "Sys":421865720, "Lookups":0, "Mallocs":13417452,"Frees":13402709, "HeapAlloc":191472672, "HeapSys":398295040, "HeapIdle":204726272, "HeapInuse":193568768,"HeapReleased":77086720, "HeapObjects":14743,"GCSys":13451264, "LastGC":1631574404279280502,"PauseTotalNs":10307006473, "PauseNs":[1522555, 1731683, 2183382, 1339856, 1450571, 825193, 1130356, 1682357, 1379171, 724470, 949354, 3239206, 1509396, 1414027, 1059970, 1997741, 1114633, 1926122, 534994, 1350355, 1285859, 1828201, 1497014, 388322, 1129436, 688231, 1135373, 1549800, 1261278, 1676880, 669277, 1456609, 1072783, 1750456, 1184433, 1538400, 622560, 1254758, 1552776, 1015347, 1085733, 898772, 1567139, 915653, 803995, 1947778, 1024808, 2409985, 740658, 1016863, 1146386, 2361719, 1603216, 1386807, 1748747, 1553573, 665440, 1529032, 1537305, 1273350, 1936419, 1480184, 951435, 929856, 1512245, 675373, 1233303, 1252736, 1053580, 1994653, 1781785, 1341758, 1831030, 1561148, 953022, 1314325, 770507, 1553906, 1557764, 1934956, 885847, 716932, 1692976, 1018133, 1296098, 1233747, 583352, 1660686, 1599502, 1088643, 1781456, 672775, 2257452, 1028403, 1285261, 861356, 1046689, 1704738, 1547189, 2344450, 1150606, 2238403, 1501275, 1713368, 1051764, 2236163, 1405552, 1043829, 1256908, 1639225, 1193784, 1419684, 1010370, 789257, 1220358, 1729911, 1513118, 1128584, 1307623, 1638934, 830572, 969125, 1438870, 651256, 1576629, 1745397, 954224, 492307, 1317108, 1297101, 1485084, 1405316, 1707648, 897376, 1457836, 1981447, 1559678, 752780, 1461583, 976529, 1440825, 1017749, 4498253, 1611363, 1553063, 1584572, 1610346, 780151, 2260906, 1728103, 1154600, 1753157, 1536509, 1735001, 905666, 1470819, 961787, 1195535, 1510929, 1323932, 1655040, 1714631, 2077962, 2618604, 1631007, 1563214, 1431306, 857753, 1529109, 2405972, 1167192, 1552101, 671525, 1318129, 1187657, 1229838, 3141504, 1895747, 1383365, 970464, 1403965, 525912, 1064892, 964926, 638968, 1160932, 1171060, 980620, 2263070, 1098895, 814910, 1211670, 809693, 770230, 1339201, 1376415, 1434169, 1581300, 560808, 1130164, 1320488, 1922465, 1213852, 1313327, 532521, 1142774, 1583647, 680322, 1069821, 1470737, 1154062, 1469167, 2480124, 1066728, 1849577, 708248, 1263306, 1789715, 1010891, 783421, 1254756, 1674556, 847684, 1391527, 1785753, 1028875, 2254788, 979388, 757267, 1323915, 927381, 1151434, 1312458, 899685, 1439036, 1415579, 1434424, 1487875, 1492447, 1108317, 1180398, 849326, 984818, 1317612, 1552251, 1668022, 586229, 683037, 704910, 1275967, 1475516, 1810629, 951259, 692225, 1507831, 1923340], "NumGC":3898} Allocated: 1674042240, active: 1674063872, metadata: 12320256 (n_thp 0), resident: 1712668672, mapped: 1726672896, retained: 1372917760 2021-09-13 16:06:50 [ss-workload] instance inse% dele% read% 2021-09-13 16:06:50 [ss-workload] 0 15% 15% 69% 2021-09-13 16:06:50 [ss-workload] 1 15% 15% 68% 2021-09-13 16:06:50 [ss-workload] 2 15% 15% 69% 2021-09-13 16:06:50 [ss-workload] 3 15% 15% 68% 2021-09-13 16:06:50 [ss-workload] 4 3% 3% 92% 2021-09-13 16:06:50 [ss-workload] 5 3% 3% 92% 2021-09-13 16:06:50 [ss-workload] 6 3% 3% 92% 2021-09-13 16:06:50 [ss-workload] 7 18% 18% 62% 2021-09-13 16:06:50 [ss-workload] 8 18% 18% 62% 2021-09-13 16:06:50 [ss-workload] 9 18% 18% 62% 2021-09-13 16:06:50 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:06:50 [ss-diag] 0 11587 11275 0 0 215 1 14092820 1 20455384 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:06:50 [ss-diag] 1 11968 11732 0 0 228 1 14944944 1 15471944 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:06:50 [ss-diag] 2 9997 9707 0 0 255 1 16714740 1 19013408 0 1.000000 1.125549 1.000000 0:0:0 2021-09-13 16:06:50 [ss-diag] 3 9593 9097 0 0 497 2 32577356 1 32521304 0 1.000000 1.125549 1.000000 0:0:0 2021-09-13 16:06:50 [ss-diag] 4 63591 63045 0 0 403 3 26415844 2 35866180 65588 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:06:50 [ss-diag] 5 54046 53644 0 0 155 1 10159940 2 26420356 65588 1.000000 1.125549 1.000000 0:0:0 2021-09-13 16:06:50 [ss-diag] 6 55885 55301 0 0 356 2 23335088 2 38358396 65604 1.000000 1.125549 1.000000 0:0:0 2021-09-13 16:06:50 [ss-diag] 7 20726 20278 0 0 437 2 28644476 1 29373240 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:06:50 [ss-diag] 8 18886 18580 0 0 318 1 20844264 1 20060512 0 1.000000 1.125549 1.000000 0:0:0 2021-09-13 16:06:50 [ss-diag] 9 17851 17315 0 0 257 1 16845836 1 35142936 0 1.000000 1.125549 1.000000 0:0:0 2021-09-13 16:07:04 [gsi-test] [] QuotaTuner: 4831838208 -> 4348654387 Mon Sep 13 16:07:12 PDT 2021 RSS = 1639108 2021-09-13T16:07:14.534-07:00 [Info] memstats {"Alloc":180537192, "TotalAlloc":466267973680, "Sys":421865720, "Lookups":0, "Mallocs":15097513,"Frees":15081465, "HeapAlloc":180537192, "HeapSys":398393344, "HeapIdle":215801856, "HeapInuse":182591488,"HeapReleased":104947712, "HeapObjects":16048,"GCSys":13418496, "LastGC":1631574434490860855,"PauseTotalNs":10961665843, "PauseNs":[1952955, 1631031, 1484004, 940648, 1753558, 1824172, 2205491, 1567853, 1880648, 1074819, 1670279, 1564054, 1108133, 1374727, 1755590, 1701802, 1403452, 1446177, 1163328, 928513, 1182266, 1797169, 2068580, 1562997, 1183959, 1068529, 1897588, 1304981, 1373808, 1970614, 2751053, 1311204, 1489224, 1237424, 1243855, 2037303, 1310691, 984560, 1715348, 1317202, 1069706, 1494044, 934466, 806460, 1757392, 1228914, 1513750, 1910594, 3649120, 1753571, 1615214, 1719961, 1659031, 827914, 3000096, 1687380, 1602905, 1667905, 1239634, 803808, 1129567, 2232031, 1347974, 1669362, 1261287, 984578, 1178577, 1229850, 2455766, 1271744, 1093677, 1541246, 1227846, 1133544, 880862, 1330867, 1593604, 927952, 1750415, 1418127, 495065, 523348, 1279441, 1689324, 1689438, 1052835, 2046962, 1011364, 2105317, 1570209, 1263248, 1136559, 2200550, 1537691, 955336, 1128069, 1035460, 1867590, 1247975, 984881, 1295528, 2500179, 1660479, 496619, 1759506, 1862144, 846885, 1340248, 1657865, 984660, 1949641, 1530987, 1588996, 1481341, 1281895, 1745033, 1346397, 1549003, 1562818, 1201901, 1964380, 1153965, 2105718, 737157, 1759853, 2003290, 1537200, 1083220, 1660503, 738592, 1918438, 1229583, 898343, 1368216, 1404152, 1469351, 1092220, 1238703, 2204784, 1122133, 3207717, 932459, 3393570, 958748, 385192, 1760882, 313106, 1504673, 986127, 871862, 1879431, 1396100, 1365614, 1180949, 792914, 2274690, 1505345, 1667079, 2017197, 1482126, 1325339, 1574659, 1756584, 1506079, 1482050, 1804658, 676457, 1200671, 2030100, 1419879, 1576258, 1518618, 1169348, 1316115, 1370251, 1612737, 1544572, 697601, 2031631, 1257825, 1144791, 982599, 1177438, 1380276, 1277865, 1114980, 1244748, 1243963, 1123116, 1632006, 508086, 1217044, 1172923, 855615, 1175368, 794179, 1418876, 1367712, 1412101, 837598, 1753570, 1285405, 1599020, 1745051, 1638956, 1677496, 1380041, 1384890, 1496397, 1438018, 1217465, 1633828, 1774544, 1526902, 1672321, 722075, 2013205, 769840, 1571607, 869743, 1523421, 1032874, 1019927, 1353269, 1508466, 1513492, 1268769, 769157, 1391900, 2042323, 1622813, 1594351, 1563289, 1821752, 1469795, 1063049, 1483260, 1422020, 1166860, 1591154, 1154253, 953202, 1311746, 1262932, 1311070, 851023, 1562242, 2009968, 2112773, 1429284, 1708253, 1022173, 1015344, 1136225, 950729, 1447333], "NumGC":4348} Allocated: 1655020416, active: 1655042048, metadata: 12320256 (n_thp 0), resident: 1704214528, mapped: 1718218752, retained: 1381371904 2021-09-13 16:07:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: starting... frag 99, data: 49152, used: 17067998 log:(0 - 17092608) head:0 2021-09-13 16:07:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: completed... frag 99, data: 49152, used: 19042270, disk: 20053246, relocated: 0, retries: 0, skipped: 191, cleaned:28496642, log:(28496642 - 48549888), disk:(28496642-48549888), activeFragRatio:30, activeMaxFragRatio:80, run:29 duration:43 ms elapsed: 300009 ms 2021-09-13T16:07:44.732-07:00 [Info] memstats {"Alloc":177354472, "TotalAlloc":517258333592, "Sys":421865720, "Lookups":0, "Mallocs":16740573,"Frees":16724815, "HeapAlloc":177354472, "HeapSys":398426112, "HeapIdle":218947584, "HeapInuse":179478528,"HeapReleased":68476928, "HeapObjects":15758,"GCSys":13385728, "LastGC":1631574464689132391,"PauseTotalNs":12091874494, "PauseNs":[1455609, 1369313, 1312380, 1928110, 1535045, 3163346, 1017874, 1043069, 4390265, 1750473, 1039748, 848264, 1697039, 953896, 1340585, 1273512, 1964024, 2013911, 1739799, 2295611, 1773524, 2024677, 1180490, 1862370, 1385830, 1272564, 1430513, 1343270, 40218083, 1691112, 1451578, 1547881, 1081798, 1723692, 848201, 1281064, 986516, 1359879, 1647089, 1314201, 2246748, 482609, 1538822, 1297719, 1388099, 1138595, 1408492, 885623, 1758445, 1592516, 1631069, 1602610, 1576035, 1952748, 1471681, 1586702, 1136069, 2476284, 1502557, 1312613, 1271009, 1480269, 1019947, 1259969, 1213945, 1768051, 1063364, 907450, 1072091, 1045901, 1198263, 755416, 1566209, 827060, 1324790, 732289, 5916817, 1874490, 1030682, 1224039, 1541450, 1269716, 1257804, 753994, 1216101, 865682, 1242368, 2311178, 611610, 1469056, 1979029, 1449004, 1712279, 2309930, 1130056, 1771663, 1668946, 1085168, 1514055, 1746959, 2358673, 612683, 1473895, 1819009, 2344144, 668356, 1125393, 1336679, 1710069, 1367606, 1565436, 739490, 2445871, 1095591, 1356437, 2440932, 672871, 2178201, 1275280, 1080819, 1689293, 1652999, 1893902, 1276948, 3828547, 5419295, 7624850, 7498903, 8895140, 12411639, 12361970, 9893473, 4567254, 8632320, 6221556, 7565371, 17613843, 11951310, 6853012, 16365239, 24481368, 10982639, 20215340, 9516238, 11017373, 3958138, 13527326, 7940605, 8200209, 4953888, 7244000, 15760269, 11738599, 21850937, 13533858, 802019, 1002558, 1160889, 1498697, 1284148, 5531245, 1569504, 3761229, 10411908, 13419881, 3293173, 10587753, 8343488, 12500712, 6652813, 11525190, 15623767, 7928921, 4438246, 7804638, 6278102, 22078741, 23422043, 1700655, 1293000, 1591366, 1542701, 1387523, 1375908, 1258219, 1592283, 935931, 1531246, 1045835, 1063069, 1554557, 1240928, 1694548, 1122786, 928470, 1459586, 972737, 977454, 1786730, 1266726, 2989002, 1677602, 1293411, 835931, 1022103, 1508310, 1412366, 1267446, 1363517, 1664461, 1290286, 1381930, 1255969, 1197797, 1162437, 1256803, 1461040, 1257411, 756878, 1421606, 1287995, 1635723, 2081107, 3124935, 1275867, 561900, 1549518, 1854981, 2088890, 1458326, 792422, 2931981, 1579629, 1249572, 1689239, 1565590, 1098417, 1791180, 923783, 1201928, 1406600, 1242070, 1472745, 1860216, 596823, 1437856, 1524287, 464706, 1966357, 1931350, 1125719, 1053437, 1694547, 1087268, 1341706, 1660107], "NumGC":4786} Allocated: 1841131872, active: 1841147904, metadata: 12320256 (n_thp 0), resident: 1884848128, mapped: 1898852352, retained: 1200738304 2021-09-13 16:08:02 [ss-workload] instance inse% dele% read% 2021-09-13 16:08:02 [ss-workload] 0 15% 15% 69% 2021-09-13 16:08:02 [ss-workload] 1 15% 15% 69% 2021-09-13 16:08:02 [ss-workload] 2 15% 15% 69% 2021-09-13 16:08:02 [ss-workload] 3 15% 15% 69% 2021-09-13 16:08:02 [ss-workload] 4 3% 3% 92% 2021-09-13 16:08:02 [ss-workload] 5 3% 3% 92% 2021-09-13 16:08:02 [ss-workload] 6 3% 3% 92% 2021-09-13 16:08:02 [ss-workload] 7 18% 18% 62% 2021-09-13 16:08:02 [ss-workload] 8 18% 18% 62% 2021-09-13 16:08:02 [ss-workload] 9 18% 18% 62% 2021-09-13 16:08:02 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:08:02 [ss-diag] 0 13559 13075 0 0 215 1 14092820 1 31735560 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:08:02 [ss-diag] 1 14238 13866 0 0 228 1 14944944 1 24386888 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:08:02 [ss-diag] 2 11873 11411 0 0 255 1 16714740 1 30293136 0 1.000000 1.190252 1.000000 0:0:0 2021-09-13 16:08:02 [ss-diag] 3 11701 11345 0 0 497 2 32577356 1 23342216 0 1.000000 1.190252 1.000000 0:0:0 2021-09-13 16:08:02 [ss-diag] 4 80462 80046 0 0 403 3 26415844 2 27340316 65588 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:08:02 [ss-diag] 5 69501 68921 0 0 155 1 10159940 2 38095596 65588 1.000000 1.190252 1.000000 0:0:0 2021-09-13 16:08:02 [ss-diag] 6 71647 71129 0 0 356 2 23335088 2 34030388 65604 1.000000 1.190252 1.000000 0:0:0 2021-09-13 16:08:02 [ss-diag] 7 25101 24597 0 0 437 2 28644476 1 33045944 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:08:02 [ss-diag] 8 23548 23050 0 0 318 1 20844264 2 32719268 65588 1.000000 1.190252 1.000000 0:0:0 2021-09-13 16:08:02 [ss-diag] 9 22458 22066 0 0 257 1 16845836 1 25702552 0 1.000000 1.190252 1.000000 0:0:0 Mon Sep 13 16:08:12 PDT 2021 RSS = 1712872 2021-09-13T16:08:14.964-07:00 [Info] memstats {"Alloc":317257200, "TotalAlloc":570892949184, "Sys":421865720, "Lookups":0, "Mallocs":18468751,"Frees":18451375, "HeapAlloc":317257200, "HeapSys":398229504, "HeapIdle":78888960, "HeapInuse":319340544,"HeapReleased":74219520, "HeapObjects":17376,"GCSys":13451264, "LastGC":1631574494963660614,"PauseTotalNs":12999053055, "PauseNs":[1152851, 1559370, 1512259, 1444071, 2666699, 1472327, 1544386, 1098352, 1914824, 1330267, 1434197, 1048839, 1797718, 1560987, 1052742, 947804, 1566969, 1590521, 1275457, 1244128, 759546, 1758208, 1567811, 1687592, 1873516, 1499326, 1145732, 1208050, 1602122, 1345167, 1789336, 866282, 1201310, 953285, 795235, 1630071, 1306156, 1779985, 926930, 3247913, 1283630, 1380940, 697939, 1669994, 1592638, 2163960, 858931, 910331, 1718006, 801205, 1169265, 626560, 1500726, 1689074, 1066638, 1129425, 1527632, 1613699, 905599, 1169431, 1307445, 2078082, 968734, 1351491, 1100715, 1054689, 826462, 2548362, 1123806, 1720122, 1410592, 1586878, 1844069, 1502007, 1922232, 1930627, 1540147, 1553977, 1163613, 1081790, 1312357, 1949265, 2043051, 1116867, 1958264, 1467524, 1579762, 2388965, 1354314, 1336488, 1383556, 899624, 866351, 1644650, 1153552, 1395867, 2126374, 1278430, 1752120, 1606471, 1183334, 1257444, 769693, 1668628, 938007, 1855562, 555794, 683180, 1372153, 1718267, 1487733, 1552728, 1329788, 2362945, 1985729, 1176791, 1914047, 1221358, 2467737, 806851, 1749327, 1868387, 1684133, 727062, 1273473, 3153662, 1174965, 993809, 1925117, 995672, 1719269, 1917137, 1121586, 1437679, 971387, 1661621, 635402, 873605, 1641193, 1345154, 1624077, 1165919, 1192278, 1241577, 1572834, 1453684, 1301464, 1669967, 1054373, 1377940, 1838481, 1691443, 1525743, 1210776, 1493431, 1488126, 1153302, 1697912, 1926469, 1457219, 1405824, 828006, 1473147, 1857242, 1475045, 1470554, 1354762, 6946622, 593399, 2192807, 995840, 1723425, 1437699, 1100428, 1474091, 938298, 1461121, 984561, 1672069, 2034841, 1839199, 1163497, 1873618, 974065, 407460, 1542973, 1111162, 1286547, 1089679, 1979366, 957591, 1686270, 1493547, 891952, 878303, 1614984, 1341814, 1135302, 1316610, 1515632, 2238966, 1401575, 1887279, 2058837, 1723934, 2307377, 1022383, 1309196, 824579, 1654979, 1424374, 1361269, 1409456, 1898504, 900077, 1041211, 974765, 1774631, 1222088, 1152952, 1148075, 1470822, 1688367, 1255553, 734799, 1411754, 1789068, 1455164, 2119198, 1203319, 1922321, 831943, 840335, 959366, 1242436, 1345735, 924759, 1587574, 1492665, 359259, 1778923, 926836, 1801361, 1800587, 1888895, 1082476, 1592900, 2073694, 1033144, 1681818, 935074, 1661407, 1173819, 932975, 1233648, 1556327], "NumGC":5251} Allocated: 1804530016, active: 1804546048, metadata: 12326144 (n_thp 0), resident: 1836519424, mapped: 1850519552, retained: 1249071104 2021-09-13T16:08:45.192-07:00 [Info] memstats {"Alloc":219434504, "TotalAlloc":628757246096, "Sys":421865720, "Lookups":0, "Mallocs":20328692,"Frees":20310924, "HeapAlloc":219434504, "HeapSys":398360576, "HeapIdle":176873472, "HeapInuse":221487104,"HeapReleased":90669056, "HeapObjects":17768,"GCSys":13320192, "LastGC":1631574525160356712,"PauseTotalNs":13717573171, "PauseNs":[1410034, 3609140, 1128683, 1469657, 1048517, 1710310, 1320556, 942048, 1693434, 1468180, 1465875, 838088, 821852, 1283574, 1278820, 845954, 497709, 1492401, 1453402, 1036407, 1449281, 1626509, 1263537, 1014569, 823403, 1737275, 888287, 872677, 1224087, 1408497, 1442990, 1469454, 1417486, 1202276, 1181700, 2861197, 881323, 883020, 1460708, 1139978, 1515312, 1641091, 1054153, 842820, 1202385, 1067304, 1359947, 1144488, 950799, 1401974, 554430, 1422113, 295841, 1433955, 1259585, 1176786, 1285810, 1435140, 1367541, 1366910, 1302938, 1166037, 1098528, 1366709, 1797963, 1149021, 1658680, 1277688, 769893, 1646497, 3522328, 1553527, 717224, 1160561, 698370, 1319009, 1572480, 947277, 1918215, 1554447, 1132875, 1465653, 1408157, 1586445, 2556929, 1257689, 1483001, 1215998, 1547604, 883728, 1433185, 1507830, 1773476, 1654101, 1012828, 804819, 1379643, 1059081, 684794, 1680574, 1209175, 940758, 48605228, 2105541, 1713297, 1368790, 956272, 700101, 1511829, 1547252, 1429770, 838315, 1855620, 1525916, 2134066, 1465896, 1695449, 495107, 1419552, 1707580, 1347674, 1616746, 1963741, 1620527, 1792006, 1042313, 838314, 1668622, 2217128, 1636242, 1069545, 1717626, 989704, 800622, 1770251, 1928476, 1394217, 526640, 1529062, 2156144, 2299701, 1682528, 1183578, 1747598, 2362770, 1579249, 1726805, 1362140, 1781760, 1764457, 1054975, 975846, 1382125, 1337723, 1795005, 1194815, 1582336, 1322002, 1273005, 1530543, 1653665, 1110329, 1636205, 1250232, 1328291, 1355024, 1450133, 1259651, 1663834, 720307, 1377281, 1035889, 1898720, 1227762, 1676576, 1343240, 374554, 1400581, 1331109, 1217423, 1430285, 916390, 1548118, 1019065, 1293060, 1113598, 1424476, 1719746, 1426588, 1256657, 1316900, 1813741, 1835742, 893916, 1266546, 1413619, 1316105, 1775444, 1391972, 1376530, 709951, 1316180, 1292436, 1646936, 1174073, 874563, 1158445, 1000451, 881288, 1155126, 1605375, 1085687, 1413243, 1661374, 2281577, 1828787, 1202576, 1440946, 1004897, 1572273, 2236544, 1269084, 1097519, 328170, 904014, 1118799, 1503916, 1052161, 252187, 3913683, 1695660, 1558401, 1239623, 921561, 2288507, 1109367, 815895, 1099505, 1027942, 998815, 1152313, 2150420, 1435812, 1166141, 892586, 1477079, 700319, 1563971, 1336428, 1041439, 631806, 771539, 961908, 1437190, 1626310, 1874368], "NumGC":5745} Allocated: 1798828384, active: 1798844416, metadata: 12326144 (n_thp 0), resident: 1833111552, mapped: 1847111680, retained: 1252478976 2021-09-13 16:09:11 [ss-workload] instance inse% dele% read% 2021-09-13 16:09:11 [ss-workload] 0 15% 15% 69% 2021-09-13 16:09:11 [ss-workload] 1 15% 15% 69% Mon Sep 13 16:09:12 PDT 2021 RSS = 2171112 2021-09-13 16:09:11 [ss-workload] 2 15% 15% 69% 2021-09-13 16:09:11 [ss-workload] 3 15% 15% 69% 2021-09-13 16:09:11 [ss-workload] 4 3% 3% 92% 2021-09-13 16:09:11 [ss-workload] 5 3% 3% 92% 2021-09-13 16:09:11 [ss-workload] 6 3% 3% 92% 2021-09-13 16:09:11 [ss-workload] 7 18% 18% 62% 2021-09-13 16:09:11 [ss-workload] 8 18% 18% 62% 2021-09-13 16:09:11 [ss-workload] 9 18% 18% 62% 2021-09-13 16:09:11 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13T16:09:15.449-07:00 [Info] memstats {"Alloc":194735096, "TotalAlloc":672263000144, "Sys":421865720, "Lookups":0, "Mallocs":21740027,"Frees":21723279, "HeapAlloc":194735096, "HeapSys":398229504, "HeapIdle":201457664, "HeapInuse":196771840,"HeapReleased":70721536, "HeapObjects":16748,"GCSys":13451264, "LastGC":1631574555405654087,"PauseTotalNs":15721303680, "PauseNs":[2270313, 1353896, 866764, 1626695, 1180412, 4032113, 1103962, 19190115, 7766459, 27321353, 15816442, 16099565, 22083597, 20632643, 20532817, 15878823, 23371884, 19118072, 30026050, 25620441, 36017907, 25010339, 28429516, 19755516, 26087001, 37782587, 28864507, 29907163, 31775079, 35964000, 35905939, 38517903, 40420963, 34019475, 46860786, 40322608, 40072601, 15229888, 16825141, 2658163, 7302736, 9538072, 9034181, 14212370, 16911105, 15094956, 18192820, 18421027, 24297226, 21546071, 21085260, 23252343, 29651229, 22203921, 17998504, 22900955, 18815568, 23506443, 30495135, 36099415, 35617134, 33935810, 39134158, 38386984, 31136346, 28883690, 37109419, 39379034, 44760283, 770563, 2822764, 689398, 635204, 1606766, 1899176, 2525973, 1796808, 1812721, 1258548, 1745390, 1443786, 1005663, 1319141, 1486897, 1046405, 856814, 735481, 1605618, 1407795, 2039793, 681535, 1399327, 1031901, 1122227, 1172782, 1429874, 2073391, 1529211, 1386094, 1064808, 1317886, 1222428, 1534298, 659149, 1189863, 2151997, 575418, 1020423, 1492654, 504914, 706453, 1348430, 768135, 1547399, 1332150, 1630262, 1245989, 931228, 748755, 2013599, 1375424, 1445064, 1425956, 1467487, 1475232, 1515709, 1288116, 1955919, 1390044, 1464577, 1337062, 943164, 622292, 1090100, 929897, 1485857, 1378602, 1031753, 1310686, 1117829, 1686209, 567753, 1485986, 899786, 1226352, 1313014, 1435850, 1701081, 1301680, 1115609, 2044200, 1591380, 1434164, 1608380, 1845037, 1716935, 1371586, 1344697, 1243846, 1330851, 1125483, 1091176, 2197210, 530634, 783285, 1632967, 1717153, 1216805, 1111809, 549667, 1221244, 2050397, 1108137, 1581713, 1728663, 1328353, 382689, 1152082, 1490926, 1070995, 1391792, 1373484, 1792887, 802270, 1149413, 1911594, 938711, 1221456, 376163, 741876, 1009976, 1425619, 1280842, 1177268, 586508, 601790, 923685, 1580215, 884482, 1544812, 417140, 1676459, 1172043, 1130140, 1722487, 1421898, 1071493, 1284207, 1278039, 1103817, 1566472, 2291285, 785869, 1173726, 1574623, 911813, 1716247, 2155568, 1499902, 1388418, 2259235, 955632, 1452844, 1303729, 1086025, 1856373, 1771490, 1698181, 979491, 1678762, 1727688, 1513215, 1552231, 1291713, 1310711, 1849584, 998152, 1628360, 1603990, 1156634, 1001138, 1094846, 1340509, 1046213, 1258599, 799143, 999832, 1672958, 1777538, 1302092, 1741347, 1551556, 1532371, 1163554, 1781936, 1362262], "NumGC":6098} Allocated: 2355004864, active: 2355032064, metadata: 14425600 (n_thp 0), resident: 2438733824, mapped: 2450632704, retained: 1118720000 2021-09-13 16:09:11 [ss-diag] 0 16118 15764 0 0 215 1 14092820 1 23210016 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:09:11 [ss-diag] 1 16078 15690 0 0 228 1 14944944 1 25440648 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:09:11 [ss-diag] 2 14030 13684 0 0 255 1 16714740 1 22684576 0 1.000000 1.228413 1.000000 0:0:0 2021-09-13 16:09:11 [ss-diag] 3 13890 13508 0 0 497 2 32577356 1 25046192 0 1.000000 1.228413 1.000000 0:0:0 2021-09-13 16:09:11 [ss-diag] 4 95558 95038 0 0 403 3 26415844 2 34161340 65588 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:09:11 [ss-diag] 5 86490 86000 0 0 155 1 10159940 2 32192644 65588 1.000000 1.228413 1.000000 0:0:0 2021-09-13 16:09:11 [ss-diag] 6 86927 86359 0 0 356 2 23335088 2 37307644 65604 1.000000 1.228413 1.000000 0:0:0 2021-09-13 16:09:11 [ss-diag] 7 29433 28977 0 0 437 2 28644476 1 29898392 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:09:11 [ss-diag] 8 26556 26080 0 0 318 1 20844264 2 31277708 65588 1.000000 1.228413 1.000000 0:0:0 2021-09-13 16:09:11 [ss-diag] 9 26798 25990 0 0 257 1 16845836 2 53045916 65588 1.000000 1.228413 1.000000 0:0:0 2021-09-13T16:09:45.662-07:00 [Info] memstats {"Alloc":232809640, "TotalAlloc":726144347688, "Sys":421865720, "Lookups":0, "Mallocs":23479004,"Frees":23462433, "HeapAlloc":232809640, "HeapSys":398262272, "HeapIdle":163086336, "HeapInuse":235175936,"HeapReleased":78970880, "HeapObjects":16571,"GCSys":13418496, "LastGC":1631574585605005318,"PauseTotalNs":16435525013, "PauseNs":[1565247, 2098048, 1837232, 1577336, 1663128, 1336746, 1767124, 1502591, 2217808, 1861603, 1410216, 1246797, 1633242, 879085, 622621, 3393156, 1074368, 805706, 1783977, 2601630, 1519441, 1467325, 2179428, 1224219, 1766309, 1614707, 2524822, 1130752, 1438327, 1168158, 1238380, 1094754, 1900616, 1110149, 1638912, 893530, 1661582, 1646673, 1162882, 2873685, 1506356, 1036570, 1115294, 1605576, 1718583, 1820783, 2140556, 870701, 1658294, 765808, 1474197, 1614798, 1577218, 856485, 1329655, 1117694, 1319439, 824525, 1640885, 1229972, 2350446, 1986058, 1624790, 1102946, 1432118, 1460679, 1608242, 1673034, 1608979, 1364691, 1249197, 1529120, 939069, 966165, 1861394, 688638, 1439620, 1735304, 1650894, 1152732, 2478358, 1552309, 1418649, 675577, 632707, 855386, 1195115, 1246226, 1555928, 1405252, 1254522, 686441, 1247666, 2761311, 1172043, 1333707, 1496680, 1428297, 1334847, 978763, 1338794, 1471886, 1340493, 1348916, 1268513, 539696, 1062851, 1124288, 1095356, 1697240, 1462346, 905309, 818125, 1855582, 1153181, 1396547, 1461308, 775506, 1702699, 1234005, 1089561, 1839963, 1535434, 2147113, 2207612, 2485374, 795220, 1568396, 1329708, 2060158, 1172227, 1426980, 931478, 1278933, 1424716, 1614067, 6825021, 1556747, 864533, 1440375, 1069861, 2852537, 346892, 1262358, 1849171, 1322094, 1783019, 742258, 926240, 1220707, 907753, 922699, 1125028, 2278966, 866762, 1184631, 1376884, 924942, 1197687, 862595, 2158203, 1420777, 1381022, 3025538, 1203326, 1377850, 976016, 673040, 1467022, 1530804, 1371460, 831930, 493022, 667971, 1221575, 601870, 453720, 1210119, 1597198, 1945649, 1219325, 1054091, 1232884, 1636071, 1612562, 2387526, 1495383, 1093297, 1749246, 1465499, 1437968, 974625, 1566238, 1281381, 777003, 1490178, 753980, 1551562, 1268524, 1346423, 1449833, 1416796, 1589032, 1223248, 1891013, 876432, 1209677, 1470512, 1670143, 1081316, 795370, 1817460, 1339464, 1157942, 1184020, 971466, 1711312, 1179265, 1378726, 1110824, 1539950, 3931073, 2485865, 1442928, 1117444, 636259, 826750, 1107390, 60020894, 2042959, 3055762, 2886423, 2090489, 2077863, 2132018, 2069812, 2106122, 1258201, 1000278, 1815817, 1785772, 1448637, 1753443, 1714649, 844182, 1547435, 1623794, 2123957, 1433185, 2371790, 843943, 1406003, 2134695, 1499622, 881013, 1307130], "NumGC":6544} Allocated: 857064896, active: 857092096, metadata: 14442624 (n_thp 0), resident: 870375424, mapped: 882257920, retained: 2687094784 Mon Sep 13 16:10:12 PDT 2021 RSS = 1048712 2021-09-13 16:10:13 [gsi-test] [] QuotaTuner: 4348654387 -> 3913788948 2021-09-13T16:10:15.912-07:00 [Info] memstats {"Alloc":234717360, "TotalAlloc":778142841152, "Sys":421865720, "Lookups":0, "Mallocs":25158158,"Frees":25141572, "HeapAlloc":234717360, "HeapSys":398295040, "HeapIdle":161292288, "HeapInuse":237002752,"HeapReleased":94683136, "HeapObjects":16586,"GCSys":13385728, "LastGC":1631574615887009948,"PauseTotalNs":17072482083, "PauseNs":[1208614, 1020243, 1264162, 974091, 1747391, 2214049, 1626210, 820745, 2342761, 1017710, 2106678, 1119608, 1283165, 1735540, 1335391, 1924503, 1307851, 2347833, 1757954, 884539, 999103, 1649539, 3150097, 942132, 1634783, 853142, 1633686, 1089877, 1765876, 1433829, 1947779, 1022116, 1573567, 1422092, 1668384, 1193632, 1647879, 1550657, 1449617, 926927, 1209253, 1634472, 1083315, 2397845, 1267487, 1687292, 1874686, 608847, 1050339, 1696707, 1328423, 1295724, 1242992, 1807101, 1661923, 1102532, 1014930, 1469777, 1730123, 1104514, 1346680, 1463666, 1577803, 2149202, 1693792, 1765865, 2323633, 1418295, 1727830, 1165146, 1567615, 1457243, 2056827, 1074753, 1085611, 949837, 1423211, 998124, 1903750, 912223, 1621583, 1626150, 1817781, 1951000, 393992, 1120433, 878500, 978088, 1534639, 1237005, 905970, 1937900, 1905327, 693317, 1432116, 1601631, 1724768, 1486560, 1934933, 688750, 1656362, 1907107, 1653377, 1822108, 1032956, 319880, 1323660, 1307837, 1578881, 1274969, 1316196, 793290, 1757575, 1552548, 1114127, 2265227, 1562203, 2229674, 991334, 1365360, 1295401, 1101840, 1391782, 2132232, 1923942, 1880774, 2425959, 1128244, 892645, 1371731, 1386153, 1049227, 2047841, 1374136, 1969160, 1445701, 1128603, 1482121, 1286459, 796215, 1463760, 1581100, 1903573, 877807, 1441069, 1643559, 1789876, 2602998, 909072, 1186830, 1218586, 343493, 1114752, 2370386, 1663064, 1005318, 1595341, 1193401, 1439854, 897495, 1218131, 1687824, 1502004, 1531391, 1596074, 1503454, 941318, 4371286, 1430211, 1385406, 1418000, 1739321, 1374742, 760227, 831019, 1615845, 1916776, 661673, 1556968, 1346172, 1867924, 1464371, 1455126, 1596545, 1213548, 1125722, 1530028, 1398810, 1245168, 1237366, 1357321, 1765840, 1275034, 983064, 1215659, 1651908, 994526, 953624, 1288091, 2652487, 1142676, 1782106, 1056724, 1051886, 243476, 1007702, 1968501, 2448310, 1520478, 967703, 1256943, 1756001, 1253721, 1459737, 1051801, 1333854, 950285, 1625718, 668483, 1254854, 1892742, 697740, 1429545, 1138374, 1585520, 1528410, 1439272, 1088248, 1673526, 1797914, 1691392, 1638175, 1568625, 868047, 1369364, 1926972, 1327969, 1874440, 1140385, 1717039, 1488067, 887602, 1337043, 2459224, 1041963, 4256256, 1369965, 1132547, 333656, 1446751, 1295595, 2232655, 887343, 1751179, 1819904, 1355324], "NumGC":6978} Allocated: 819820000, active: 819847168, metadata: 14471328 (n_thp 0), resident: 867917824, mapped: 879800320, retained: 2689552384 2021-09-13 16:10:20 [ss-workload] instance inse% dele% read% 2021-09-13 16:10:20 [ss-workload] 0 15% 15% 69% 2021-09-13 16:10:20 [ss-workload] 1 15% 15% 69% 2021-09-13 16:10:20 [ss-workload] 2 15% 15% 69% 2021-09-13 16:10:20 [ss-workload] 3 15% 15% 69% 2021-09-13 16:10:20 [ss-workload] 4 3% 3% 92% 2021-09-13 16:10:20 [ss-workload] 5 3% 3% 92% 2021-09-13 16:10:20 [ss-workload] 6 3% 3% 92% 2021-09-13 16:10:22 [gsi-test] [6] CreateRecoveryPoint() - DO!!! 2021-09-13 16:10:22 [gsi-test] [5] CreateRecoveryPoint() - DO!!! 2021-09-13 16:10:23 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 16:10:23 [gsi-test] [9] CreateRecoveryPoint() - DO!!! 2021-09-13 16:10:23 [gsi-test] [8] CreateRecoveryPoint() - DO!!! 2021-09-13 16:10:23 [gsi-test] [2] CreateRecoveryPoint() - DO!!! 2021-09-13 16:10:23 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 16:10:24 [gsi-test] [3] CreateRecoveryPoint() - DO!!! 2021-09-13 16:10:24 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 16:10:24 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13 16:10:20 [ss-workload] 7 18% 18% 62% 2021-09-13 16:10:20 [ss-workload] 8 18% 18% 62% 2021-09-13 16:10:20 [ss-workload] 9 18% 18% 62% 2021-09-13 16:10:20 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:10:20 [ss-diag] 0 18339 17999 0 0 215 1 14092820 1 22292136 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: starting... frag 99, data: 8149982, used: 2526220258 log:(29069342 - 2555289600) head:0 2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: completed... frag 99, data: 48212658, used: 8010507438, disk: 8032790657, relocated: 34, retries: 2230, skipped: 1, cleaned:4156975924, log:(4186045266 - 12196552704), disk:(4163762047-12196552704), activeFragRatio:30, activeMaxFragRatio:80, run:4 duration:357766 ms elapsed: 471569 ms 2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:10:34 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : All daemons stopped 2021-09-13 16:10:34 [gsi-test] [0] Stop and verify... 2021-09-13 16:10:34 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 241, "num_active_sctxs": 102, "num_free_sctxs": 139, "num_trim_sctxs": 65, "num_wctxs": 324, "num_free_wctxs": 252, "num_active_wctxs": 72, "num_trim_wctxs": 36, "num_readers": 114, "num_writers": 119, "cpu": 9.89606, "total_core": 16, "threads": 45, "assigned_quota": 3913788948, "current_quota": 3913788948, "memory_resident": 0, "memory_jemalloc": 1771139072, "memory_jemalloc_allocated": 1651653280, "memory_in_use": 1707229184, "memory_stats_size_total": 1409843332, "memory_stats_size_global": 346170416, "memory_stats_size_page": 345778344, "memory_stats_size_index": 327956, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 44760, "reclaim_pending": 339926640, "buf_memused": 723810392, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 23, "num_active_smr": 6, "num_pending_smr": 6, "num_pages": 15, "items_count": 2080, "total_records": 5269, "total_ops": 242269, "compacts": 1205, "purges": 0, "lss_data_size": 59870571, "lss_used_space": 8122186908, "lss_fragmentation": 99, "lss_num_reads": 1440, "lss_read_bs": 6525085600, "lss_blk_read_bs": 6729494528, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 14635401216, "bytes_incoming": 15880182864, "write_amp_avg": 0.55, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 0 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 5269, "resident_ratio": 100.00000, "cleaner_num_reads": 1440, "cleaner_read_bs": 6525085600, "cleaner_blk_read_bs": 6729494528, "recovery_num_lssctx": 5, "recovery_num_sctxs": 25, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 5, "recovery_num_trim_sctxs": 5, "recovery_buf_memused": 1048576, "recovery_data_size": 278936, "recovery_used_space": 69824478, "recovery_lss_blk_read_bs": 245764096, "recovery_lss_read_bs": 42773848, "recovery_lss__num_reads": 700, "recovery_bytes_written": 99332096, "recovery_lss_frag": 99, "recovery_cleaner_num_reads": 700, "recovery_cleaner_read_bs": 42773848, "recovery_cleaner_blk_read_bs": 245764096, "mvcc_purge_ratio": 2.53317 }2021-09-13 16:10:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:10:20 [ss-diag] 1 18669 18327 0 0 228 1 14944944 1 22422320 0 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:10:20 [ss-diag] 2 16268 15910 0 0 255 1 16714740 1 23473328 0 1.000000 1.417542 1.000000 0:0:0 2021-09-13 16:10:36 [gsi-test] [0] Stop and verify... OK 2021-09-13 16:10:36 [gsi-test] [1] Stop and verify... 2021-09-13 16:10:20 [ss-diag] 3 16186 15852 0 0 497 2 32577356 1 21898768 0 1.000000 1.417542 1.000000 0:0:0 2021-09-13 16:10:39 [gsi-test] [1] Stop and verify... OK 2021-09-13 16:10:39 [gsi-test] [2] Stop and verify... 2021-09-13 16:10:40 [gsi-test] [2] Stop and verify... OK 2021-09-13 16:10:40 [gsi-test] [3] Stop and verify... 2021-09-13 16:10:43 [gsi-test] [3] Stop and verify... OK 2021-09-13 16:10:43 [gsi-test] [4] Stop and verify... 2021-09-13T16:10:46.128-07:00 [Info] memstats {"Alloc":185960976, "TotalAlloc":803754613128, "Sys":629883128, "Lookups":0, "Mallocs":25978968,"Frees":25964524, "HeapAlloc":185960976, "HeapSys":597393408, "HeapIdle":409346048, "HeapInuse":188047360,"HeapReleased":119275520, "HeapObjects":14444,"GCSys":20019200, "LastGC":1631574646078303673,"PauseTotalNs":19973687618, "PauseNs":[1333339, 1589017, 1400945, 1224253, 1819620, 351841, 1876181, 1468907, 1240945, 1114049, 1963010, 1582545, 1414287, 1715011, 1646543, 1192199, 1466659, 1776216, 2035292, 1824158, 1287502, 1035240, 1436866, 1567135, 784535, 1790284, 915339, 1892183, 1162442, 1365266, 2134944, 1575222, 1677039, 1216583, 1223762, 1016623, 1612819, 1630759, 1369989, 2046921, 1853297, 1451414, 1371392, 1131221, 2114204, 1599073, 1347298, 1697524, 1435958, 1108540, 1323770, 1196885, 1587717, 939546, 1502963, 1485557, 1097573, 1656053, 1772801, 1617377, 1307851, 1402851, 774482, 1192278, 2230678, 1524828, 1272352, 1029953, 1534978, 1630985, 1644011, 603803, 609465, 1652810, 957376, 843321, 812348, 1077438, 1621199, 1429316, 907846, 1844763, 1785884, 1551951, 1528075, 861928, 1282205, 2731949, 1479544, 943107, 1408066, 1913328, 1335882, 1414587, 939322, 1249678, 1644798, 1886535, 1369489, 2009456, 1611891, 626445, 1508950, 2250824, 3044899, 2852120, 4161497, 7857967, 6511896, 6001037, 5876293, 7103334, 5434886, 8755424, 8613894, 7996930, 9242416, 10926322, 10961666, 18881841, 22938772, 16631506, 18352741, 13146500, 20265943, 17823978, 27706880, 24068556, 24653852, 22247545, 29271161, 37774665, 32096927, 34235685, 30081181, 39279323, 27423489, 28046602, 51530583, 28445244, 20460226, 33819391, 41785760, 27928629, 44307571, 48026219, 40248015, 81457612, 47263574, 41333398, 43377348, 30404745, 27943817, 33831325, 42836414, 46406161, 46710938, 22365543, 51228066, 38935448, 104213808, 109841017, 22325413, 91060244, 42445470, 25701545, 29942585, 54098870, 48223614, 38338982, 51258146, 46926932, 49340529, 33699323, 67672080, 33997482, 67605313, 39922764, 67589747, 31401702, 101087754, 18535347, 31736022, 30562326, 33459306], "NumGC":7163} Allocated: 2188336000, active: 2188365824, metadata: 14727840 (n_thp 0), resident: 2364239872, mapped: 2382155776, retained: 1313026048 2021-09-13 16:10:51 [gsi-test] [] { "memory_quota": 3913788948, "count": 8562, "compacts": 42, "purges": 0, "splits": 0, "merges": 0, "inserts": 8562, "deletes": 0, "compact_conflicts": 117, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 2099, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 2, "memory_size": 32913040, "memory_size_index": 0, "allocated": 1242431520, "freed": 1209518480, "reclaimed": 1209518480, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 0, "freed_index": 0, "reclaimed_index": 0, "num_pages": 1, "items_count": 208, "total_records": 502, "num_rec_allocs": 18948, "num_rec_frees": 18446, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 561221976, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 819200, "est_recovery_size": 12438, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 25358, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 16796, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 2.41346, "currSn": 607, "gcSn": 602, "gcSnIntervals": "[0 581 603]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 26, "num_trim_wctxs": 4, "num_readers": 9, "num_writers": 12, "page_bytes": 14092820, "page_cnt": 1, "page_itemcnt": 215, "avg_item_size": 65548, "avg_page_size": 14092820, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 27538600, "page_bytes_marshalled":20844416, "page_bytes_compressed":11346209, "compression_ratio": 1.83713, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 4896, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 655480, "lss_data_size": 16448, "lss_used_space": 31145984, "lss_disk_size": 31145984, "lss_fragmentation": 99, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "bytes_written": 31145984, "bytes_incoming": 561221976, "write_amp": 0.00, "write_amp_avg": 0.06, "lss_gc_num_reads": 0, "lss_gc_reads_bs": 0, "lss_blk_gc_reads_bs": 0, "lss_gc_status": "", "lss_head_offset": 0, "lss_tail_offset": 31133696, "num_sctxs": 36, "num_free_sctxs": 26, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:10:52 [gsi-test] [4] Stop and verify... OK 2021-09-13 16:10:52 [gsi-test] [5] Stop and verify... 2021-09-13 16:10:20 [ss-diag] 4 113615 112513 0 0 403 3 26415844 3 72379448 131176 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:10:53 [gsi-test] [5] Stop and verify... OK 2021-09-13 16:10:53 [gsi-test] [6] Stop and verify... 2021-09-13 16:10:20 [ss-diag] 5 104094 103250 0 0 155 1 10159940 3 55465456 131176 1.000000 1.417542 1.000000 0:0:0 2021-09-13 16:10:53 [gsi-test] [6] Stop and verify... OK 2021-09-13 16:10:53 [gsi-test] [7] Stop and verify... 2021-09-13 16:10:20 [ss-diag] 6 106956 105826 0 0 356 2 23335088 3 74220680 131208 1.000000 1.417542 1.000000 0:0:0 2021-09-13 16:10:54 [gsi-test] [7] Stop and verify... OK 2021-09-13 16:10:54 [gsi-test] [8] Stop and verify... 2021-09-13 16:10:55 [gsi-test] [] { "memory_quota": 3913788948, "count": 8646, "compacts": 44, "purges": 0, "splits": 1, "merges": 0, "inserts": 8646, "deletes": 0, "compact_conflicts": 104, "split_conflicts": 2, "merge_conflicts": 0, "insert_conflicts": 1961, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 1, "memory_size": 27468324, "memory_size_index": 65588, "allocated": 1287602612, "freed": 1260134288, "reclaimed": 1260134288, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 65588, "freed_index": 0, "reclaimed_index": 0, "num_pages": 2, "items_count": 208, "total_records": 418, "num_rec_allocs": 19635, "num_rec_frees": 19217, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 566728008, "lss_data_size": 14849777, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 54999174, "est_recovery_size": 143626, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 25675, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 17029, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 2.00962, "currSn": 600, "gcSn": 597, "gcSnIntervals": "[0 579 598]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 32, "num_trim_wctxs": 4, "num_readers": 11, "num_writers": 13, "page_bytes": 14944944, "page_cnt": 1, "page_itemcnt": 228, "avg_item_size": 65548, "avg_page_size": 14944944, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 26489288, "page_bytes_marshalled":52307546, "page_bytes_compressed":28472064, "compression_ratio": 1.83715, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 400, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 178247181, "lss_data_size": 14989201, "lss_used_space": 56659968, "lss_disk_size": 56659968, "lss_fragmentation": 73, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "bytes_written": 56659968, "bytes_incoming": 566728008, "write_amp": 0.00, "write_amp_avg": 0.10, "lss_gc_num_reads": 0, "lss_gc_reads_bs": 0, "lss_blk_gc_reads_bs": 0, "lss_gc_status": "", "lss_head_offset": 0, "lss_tail_offset": 56647680, "num_sctxs": 38, "num_free_sctxs": 30, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:10:55 [gsi-test] [8] Stop and verify... OK 2021-09-13 16:10:55 [gsi-test] [9] Stop and verify... 2021-09-13 16:10:20 [ss-diag] 7 35369 34763 0 0 437 2 28644476 2 39797588 65588 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:10:56 [gsi-test] [] { "memory_quota": 3913788948, "count": 7640, "compacts": 39, "purges": 0, "splits": 1, "merges": 0, "inserts": 7640, "deletes": 0, "compact_conflicts": 99, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 1714, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 2, "memory_size": 30614948, "memory_size_index": 65588, "allocated": 1169831564, "freed": 1139216616, "reclaimed": 1139216616, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 65588, "freed_index": 0, "reclaimed_index": 0, "num_pages": 2, "items_count": 208, "total_records": 466, "num_rec_allocs": 17839, "num_rec_frees": 17373, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 500786720, "lss_data_size": 16634124, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 1663412400, "est_recovery_size": 143710, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 22869, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 15229, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 2.24038, "currSn": 497, "gcSn": 494, "gcSnIntervals": "[0 466 495]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 28, "num_trim_wctxs": 7, "num_readers": 11, "num_writers": 10, "page_bytes": 16714740, "page_cnt": 1, "page_itemcnt": 255, "avg_item_size": 65548, "avg_page_size": 16714740, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 17046056, "page_bytes_marshalled":50865442, "page_bytes_compressed":27686467, "compression_ratio": 1.83720, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 320, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 899307029, "lss_data_size": 42659615, "lss_used_space": 8492471436, "lss_disk_size": 8547274752, "lss_fragmentation": 99, "lss_num_reads": 1197, "lss_read_bs": 4227881963, "lss_blk_read_bs": 4431826944, "bytes_written": 12708024320, "bytes_incoming": 10129525728, "write_amp": 1.42, "write_amp_avg": 1.25, "lss_gc_num_reads": 1197, "lss_gc_reads_bs": 4227881963, "lss_blk_gc_reads_bs": 4431826944, "lss_gc_status": "frag 99, data: 48212658, used: 8010507438, disk: 8032790657, relocated: 34, retries: 2230, skipped: 1, cleaned:4156975924, log:(4186045266 - 12196552704), disk:(4163762047-12196552704), activeFragRatio:30, activeMaxFragRatio:80, run:4 duration:357766 ms elapsed: 471569 ms", "lss_head_offset": 4186045266, "lss_tail_offset": 12620279808, "num_sctxs": 96, "num_free_sctxs": 88, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:10:56 [gsi-test] [] { "memory_quota": 3913788948, "count": 7500, "compacts": 38, "purges": 0, "splits": 1, "merges": 0, "inserts": 7500, "deletes": 0, "compact_conflicts": 119, "split_conflicts": 2, "merge_conflicts": 0, "insert_conflicts": 1969, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 1, "memory_size": 31141348, "memory_size_index": 65588, "allocated": 1126428956, "freed": 1095287608, "reclaimed": 1095287608, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 65588, "freed_index": 0, "reclaimed_index": 0, "num_pages": 2, "items_count": 208, "total_records": 474, "num_rec_allocs": 17177, "num_rec_frees": 16703, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 491610000, "lss_data_size": 14742323, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 1474232300, "est_recovery_size": 143710, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 22359, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 14859, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 2.27885, "currSn": 492, "gcSn": 490, "gcSnIntervals": "[0 468 491]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 30, "num_trim_wctxs": 3, "num_readers": 12, "num_writers": 10, "page_bytes": 32577356, "page_cnt": 2, "page_itemcnt": 497, "avg_item_size": 65548, "avg_page_size": 16288678, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 19012288, "page_bytes_marshalled":52110974, "page_bytes_compressed":28363158, "compression_ratio": 1.83728, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 2760, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 899307029, "lss_data_size": 42659615, "lss_used_space": 8492471436, "lss_disk_size": 8547274752, "lss_fragmentation": 99, "lss_num_reads": 1197, "lss_read_bs": 4227881963, "lss_blk_read_bs": 4431826944, "bytes_written": 12708024320, "bytes_incoming": 10139620120, "write_amp": 1.42, "write_amp_avg": 1.24, "lss_gc_num_reads": 1197, "lss_gc_reads_bs": 4227881963, "lss_blk_gc_reads_bs": 4431826944, "lss_gc_status": "frag 99, data: 48212658, used: 8010507438, disk: 8032790657, relocated: 34, retries: 2230, skipped: 1, cleaned:4156975924, log:(4186045266 - 12196552704), disk:(4163762047-12196552704), activeFragRatio:30, activeMaxFragRatio:80, run:4 duration:357766 ms elapsed: 471569 ms", "lss_head_offset": 4186045266, "lss_tail_offset": 12620279808, "num_sctxs": 96, "num_free_sctxs": 87, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:10:56 [gsi-test] [] { "memory_quota": 3913788948, "count": 60574, "compacts": 301, "purges": 0, "splits": 2, "merges": 0, "inserts": 60574, "deletes": 0, "compact_conflicts": 236, "split_conflicts": 4, "merge_conflicts": 0, "insert_conflicts": 2402, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 2, "memory_size": 90220664, "memory_size_index": 131176, "allocated": 7515131284, "freed": 7424910620, "reclaimed": 7424910620, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 131176, "freed_index": 0, "reclaimed_index": 0, "num_pages": 3, "items_count": 208, "total_records": 1374, "num_rec_allocs": 114453, "num_rec_frees": 113079, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 3970504552, "lss_data_size": 20485322, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 136568813, "est_recovery_size": 684710, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 697773, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 637199, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 6.60577, "currSn": 9963, "gcSn": 9942, "gcSnIntervals": "[0 9651 9943]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 30, "num_trim_wctxs": 1, "num_readers": 16, "num_writers": 8, "page_bytes": 26415844, "page_cnt": 3, "page_itemcnt": 403, "avg_item_size": 65548, "avg_page_size": 8805281, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 31340672, "page_bytes_marshalled":177373496, "page_bytes_compressed":96543576, "compression_ratio": 1.83724, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 19456, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 369107594, "lss_data_size": 20690294, "lss_used_space": 143669264, "lss_disk_size": 159629312, "lss_fragmentation": 85, "lss_num_reads": 243, "lss_read_bs": 2297203637, "lss_blk_read_bs": 2297667584, "bytes_written": 2441330688, "bytes_incoming": 3970504552, "write_amp": 0.00, "write_amp_avg": 0.61, "lss_gc_num_reads": 243, "lss_gc_reads_bs": 2297203637, "lss_blk_gc_reads_bs": 2297667584, "lss_gc_status": "frag 0, data: 16566588, used: 10713104, disk: 2308374528, relocated: 3, retries: 232, skipped: 0, cleaned:2267749018, log:(2297661424 - 2308374528), disk:(0-2308374528), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:1922 ms elapsed: 1922 ms", "lss_head_offset": 2297661424, "lss_tail_offset": 2424799232, "num_sctxs": 41, "num_free_sctxs": 31, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:10:56 [gsi-test] [] { "memory_quota": 3913788948, "count": 55970, "compacts": 279, "purges": 0, "splits": 2, "merges": 0, "inserts": 55970, "deletes": 0, "compact_conflicts": 275, "split_conflicts": 3, "merge_conflicts": 0, "insert_conflicts": 2194, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 10, "memory_size": 60714736, "memory_size_index": 131176, "allocated": 6878958772, "freed": 6818244036, "reclaimed": 6818244036, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 131176, "freed_index": 0, "reclaimed_index": 0, "num_pages": 3, "items_count": 208, "total_records": 924, "num_rec_allocs": 104772, "num_rec_frees": 103848, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 3668721560, "lss_data_size": 10750004, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 1075000400, "est_recovery_size": 471624, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 640982, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 585012, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 4.44231, "currSn": 8177, "gcSn": 8151, "gcSnIntervals": "[0 7887 8152]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 36, "num_trim_wctxs": 0, "num_readers": 16, "num_writers": 13, "page_bytes": 10159940, "page_cnt": 1, "page_itemcnt": 155, "avg_item_size": 65548, "avg_page_size": 10159940, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 39342376, "page_bytes_marshalled":234073770, "page_bytes_compressed":127409995, "compression_ratio": 1.83717, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12296, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 899307029, "lss_data_size": 42659615, "lss_used_space": 8492471436, "lss_disk_size": 8547274752, "lss_fragmentation": 99, "lss_num_reads": 1197, "lss_read_bs": 4227881963, "lss_blk_read_bs": 4431826944, "bytes_written": 12708024320, "bytes_incoming": 10139620120, "write_amp": 1.42, "write_amp_avg": 1.24, "lss_gc_num_reads": 1197, "lss_gc_reads_bs": 4227881963, "lss_blk_gc_reads_bs": 4431826944, "lss_gc_status": "frag 99, data: 48212658, used: 8010507438, disk: 8032790657, relocated: 34, retries: 2230, skipped: 1, cleaned:4156975924, log:(4186045266 - 12196552704), disk:(4163762047-12196552704), activeFragRatio:30, activeMaxFragRatio:80, run:4 duration:357766 ms elapsed: 471569 ms", "lss_head_offset": 4186045266, "lss_tail_offset": 12620279808, "num_sctxs": 96, "num_free_sctxs": 86, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:10:57 [gsi-test] [] { "memory_quota": 3913788948, "count": 55484, "compacts": 276, "purges": 0, "splits": 2, "merges": 0, "inserts": 55484, "deletes": 0, "compact_conflicts": 293, "split_conflicts": 27, "merge_conflicts": 0, "insert_conflicts": 2405, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 8, "memory_size": 77500752, "memory_size_index": 131208, "allocated": 7052928532, "freed": 6975427780, "reclaimed": 6975427780, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 131208, "freed_index": 0, "reclaimed_index": 0, "num_pages": 3, "items_count": 208, "total_records": 1180, "num_rec_allocs": 107424, "num_rec_frees": 106244, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 3636865232, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 819200, "est_recovery_size": 471624, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 634495, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 579011, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 5.67308, "currSn": 7991, "gcSn": 7968, "gcSnIntervals": "[0 7609 7969]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 36, "num_trim_wctxs": 3, "num_readers": 16, "num_writers": 15, "page_bytes": 23335088, "page_cnt": 2, "page_itemcnt": 356, "avg_item_size": 65548, "avg_page_size": 11667544, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 34619256, "page_bytes_marshalled":174227612, "page_bytes_compressed":94832788, "compression_ratio": 1.83721, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 16928, "num_total_smr": 1, "num_active_smr": 4, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 899307029, "lss_data_size": 42659615, "lss_used_space": 8492471436, "lss_disk_size": 8547274752, "lss_fragmentation": 99, "lss_num_reads": 1197, "lss_read_bs": 4227881963, "lss_blk_read_bs": 4431826944, "bytes_written": 12708024320, "bytes_incoming": 10141717656, "write_amp": 1.42, "write_amp_avg": 1.24, "lss_gc_num_reads": 1197, "lss_gc_reads_bs": 4227881963, "lss_blk_gc_reads_bs": 4431826944, "lss_gc_status": "frag 99, data: 48212658, used: 8010507438, disk: 8032790657, relocated: 34, retries: 2230, skipped: 1, cleaned:4156975924, log:(4186045266 - 12196552704), disk:(4163762047-12196552704), activeFragRatio:30, activeMaxFragRatio:80, run:4 duration:357766 ms elapsed: 471569 ms", "lss_head_offset": 4186045266, "lss_tail_offset": 12620279808, "num_sctxs": 96, "num_free_sctxs": 84, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:10:57 [gsi-test] [9] Stop and verify... OK 2021-09-13 16:10:57 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper started 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : all deamons started 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : all deamons started 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : all deamons started 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:10:57 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper started 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : all deamons started 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : all deamons started 2021-09-13 16:10:57 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper started 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : all deamons started 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : all deamons started 2021-09-13 16:10:57 [gsi-test] [] Stop and verify... OK 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:10:57 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:10:58 [plasma] [i] Plasma: Adjusting SMO limits: Avg - Item Size 65548 Avg Page Size 11745511: Old MaxPageItems 400 MinPageItems 25 MaxDeltaChainLen 200 -> New MaxPageItems 4 MinPageItems 1 MaxDeltaChainLen 2 2021-09-13 16:10:58 [plasma] [i] Plasma: Adjusting SMO limits: Avg - Item Size 65548 Avg Page Size 18366229: Old MaxPageItems 400 MinPageItems 25 MaxDeltaChainLen 200 -> New MaxPageItems 4 MinPageItems 1 MaxDeltaChainLen 2 2021-09-13 16:10:58 [plasma] [i] Plasma: Adjusting SMO limits: Avg - Item Size 65548 Avg Page Size 10666793: Old MaxPageItems 400 MinPageItems 25 MaxDeltaChainLen 200 -> New MaxPageItems 4 MinPageItems 1 MaxDeltaChainLen 2 2021-09-13 16:10:58 [plasma] [i] Plasma: Adjusting SMO limits: Avg - Item Size 65548 Avg Page Size 11605445: Old MaxPageItems 400 MinPageItems 25 MaxDeltaChainLen 200 -> New MaxPageItems 4 MinPageItems 1 MaxDeltaChainLen 2 2021-09-13 16:10:58 [plasma] [i] Plasma: Adjusting SMO limits: Avg - Item Size 65548 Avg Page Size 16805778: Old MaxPageItems 400 MinPageItems 25 MaxDeltaChainLen 200 -> New MaxPageItems 4 MinPageItems 1 MaxDeltaChainLen 2 2021-09-13 16:10:20 [ss-diag] 8 31357 30631 0 0 7396 32 484793008 2 47664372 65588 1.000000 1.417542 1.000000 0:0:0 2021-09-13 16:10:59 [gsi-test] [] { "memory_quota": 3913788948, "count": 14976, "compacts": 74, "purges": 0, "splits": 1, "merges": 0, "inserts": 14976, "deletes": 0, "compact_conflicts": 293, "split_conflicts": 1, "merge_conflicts": 0, "insert_conflicts": 4167, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 2, "memory_size": 49635788, "memory_size_index": 65588, "allocated": 2329098356, "freed": 2279462568, "reclaimed": 2279462568, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 65588, "freed_index": 0, "reclaimed_index": 0, "num_pages": 2, "items_count": 208, "total_records": 756, "num_rec_allocs": 35519, "num_rec_frees": 34763, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 981646848, "lss_data_size": 16277238, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 52507219, "est_recovery_size": 147732, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 39430, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 24454, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 3.63462, "currSn": 690, "gcSn": 687, "gcSnIntervals": "[0 665 688]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 13, "num_trim_wctxs": 21, "num_readers": 3, "num_writers": 2, "page_bytes": 753015424, "page_cnt": 41, "page_itemcnt": 11488, "avg_item_size": 65548, "avg_page_size": 18366229, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 20192392, "page_bytes_marshalled":63384946, "page_bytes_compressed":34501168, "compression_ratio": 1.83718, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12320, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 126572599, "lss_data_size": 16416662, "lss_used_space": 53129643, "lss_disk_size": 83468288, "lss_fragmentation": 69, "lss_num_reads": 5, "lss_read_bs": 48558601, "lss_blk_read_bs": 48574464, "bytes_written": 83468288, "bytes_incoming": 981646848, "write_amp": 0.00, "write_amp_avg": 0.08, "lss_gc_num_reads": 5, "lss_gc_reads_bs": 48558601, "lss_blk_gc_reads_bs": 48574464, "lss_gc_status": "frag 75, data: 16277238, used: 66920448 log:(0 - 66920448) head:0", "lss_head_offset": 30338645, "lss_tail_offset": 83189760, "num_sctxs": 36, "num_free_sctxs": 28, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:10:59 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: starting... frag 75, data: 16277238, used: 66920448 log:(0 - 66920448) head:0 2021-09-13 16:10:59 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: completed... frag 0, data: 16277238, used: 16270002, disk: 83189760, relocated: 0, retries: 0, skipped: 1, cleaned:66919758, log:(66919758 - 83189760), disk:(0-83189760), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:643 ms elapsed: 643 ms 2021-09-13 16:10:59 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 20167 KB (expected = 10240 KB), wCtxCnt = 20, objCnt 1, changed reclaimList flush threshold from 50 to 0, changed reclaimSize flush threshold from 10240 KB to 409 KB. 2021-09-13 16:11:00 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 95, data: 795084, used: 17387520 log:(0 - 17387520) head:0 2021-09-13 16:11:00 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 51, data: 795084, used: 1654320, disk: 17387520, relocated: 0, retries: 0, skipped: 3, cleaned:14746064, log:(14746064 - 17387520), disk:(0-17387520), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:40 ms elapsed: 40 ms 2021-09-13 16:11:00 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 26381 KB (expected = 10240 KB), wCtxCnt = 22, objCnt 1, changed reclaimList flush threshold from 50 to 0, changed reclaimSize flush threshold from 10240 KB to 372 KB. 2021-09-13 16:11:01 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 18246 KB (expected = 10240 KB), wCtxCnt = 18, objCnt 4, changed reclaimList flush threshold from 50 to 0, changed reclaimSize flush threshold from 10240 KB to 455 KB. 2021-09-13 16:11:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: starting... frag 99, data: 16566588, used: 2267751082 log:(29912406 - 2297663488) head:0 2021-09-13 16:11:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: completed... frag 68, data: 23700362, used: 74660555, disk: 201795600, relocated: 7, retries: 465, skipped: 5, cleaned:2394883976, log:(2424796382 - 2499457024), disk:(2297661424-2499457024), activeFragRatio:30, activeMaxFragRatio:80, run:2 duration:5417 ms elapsed: 498259 ms 2021-09-13 16:10:20 [ss-diag] 9 33529 32965 0 0 9581 41 628015388 34 39140812 2164548 1.000000 1.417542 1.000000 0:0:0 2021-09-13 16:11:02 [gsi-test] [] { "memory_quota": 3913788948, "count": 14460, "compacts": 72, "purges": 0, "splits": 1, "merges": 0, "inserts": 14460, "deletes": 0, "compact_conflicts": 265, "split_conflicts": 1, "merge_conflicts": 0, "insert_conflicts": 3912, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 4, "memory_size": 49894364, "memory_size_index": 65588, "allocated": 2059253204, "freed": 2009358840, "reclaimed": 2009358840, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 65588, "freed_index": 0, "reclaimed_index": 0, "num_pages": 2, "items_count": 208, "total_records": 760, "num_rec_allocs": 31391, "num_rec_frees": 30631, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 947824080, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 409600, "est_recovery_size": 270746, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 38023, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 23563, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 3.65385, "currSn": 553, "gcSn": 548, "gcSnIntervals": "[0 508 549]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 28, "num_trim_wctxs": 10, "num_readers": 8, "num_writers": 12, "page_bytes": 463948744, "page_cnt": 31, "page_itemcnt": 7078, "avg_item_size": 65548, "avg_page_size": 14966088, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 36458616, "page_bytes_marshalled":71120050, "page_bytes_compressed":38712779, "compression_ratio": 1.83712, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 7792, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 926400425, "lss_data_size": 109715686, "lss_used_space": 8647059127, "lss_disk_size": 8878825472, "lss_fragmentation": 98, "lss_num_reads": 1566, "lss_read_bs": 4404969106, "lss_blk_read_bs": 4612329472, "bytes_written": 13039575040, "bytes_incoming": 10168199048, "write_amp": 1.42, "write_amp_avg": 1.27, "lss_gc_num_reads": 1566, "lss_gc_reads_bs": 4404969106, "lss_blk_gc_reads_bs": 4612329472, "lss_gc_status": "frag 99, data: 42151027, used: 8434234542 log:(4186045266 - 12620279808) head:4186045266", "lss_head_offset": 4341598117, "lss_tail_offset": 12932399104, "num_sctxs": 99, "num_free_sctxs": 78, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:11:05 [gsi-test] [] { "memory_quota": 3913788948, "count": 13740, "compacts": 144, "purges": 0, "splits": 64, "merges": 0, "inserts": 13740, "deletes": 0, "compact_conflicts": 310, "split_conflicts": 23, "merge_conflicts": 0, "insert_conflicts": 3956, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 1, "memory_size": 41045120, "memory_size_index": 4197888, "allocated": 2255587000, "freed": 2214541880, "reclaimed": 2212050600, "reclaim_pending": 2491280, "reclaim_list_size": 2491280, "reclaim_list_count": 1, "reclaim_threshold": 0, "allocated_index": 4197888, "freed_index": 0, "reclaimed_index": 0, "num_pages": 65, "items_count": 207, "total_records": 562, "num_rec_allocs": 34195, "num_rec_frees": 33633, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 900629520, "lss_data_size": 22928413, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 1146420650, "est_recovery_size": 16206218, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 36198, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 22458, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 2.71498, "currSn": 546, "gcSn": 542, "gcSnIntervals": "[0 504 543]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 18, "num_trim_wctxs": 11, "num_readers": 3, "num_writers": 11, "page_bytes": 634307996, "page_cnt": 49, "page_itemcnt": 9677, "avg_item_size": 65548, "avg_page_size": 12945061, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 20456232, "page_bytes_marshalled":251968366, "page_bytes_compressed":137175283, "compression_ratio": 1.83684, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 7600, "num_total_smr": 1, "num_active_smr": 4, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 954094552, "lss_data_size": 127698221, "lss_used_space": 7263099714, "lss_disk_size": 8143958016, "lss_fragmentation": 98, "lss_num_reads": 1736, "lss_read_bs": 5993486785, "lss_blk_read_bs": 6204092416, "bytes_written": 13244231680, "bytes_incoming": 10180259880, "write_amp": 1.42, "write_amp_avg": 1.29, "lss_gc_num_reads": 1736, "lss_gc_reads_bs": 5993486785, "lss_blk_gc_reads_bs": 6204092416, "lss_gc_status": "frag 99, data: 42151027, used: 8434234542 log:(4186045266 - 12620279808) head:4186045266", "lss_head_offset": 5930214170, "lss_tail_offset": 13106298880, "num_sctxs": 99, "num_free_sctxs": 84, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:11:08 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 65, data: 8794136, used: 25235456 log:(0 - 25235456) head:0 2021-09-13 16:11:08 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 45, data: 8925268, used: 16527220, disk: 19885940, relocated: 44, retries: 0, skipped: 47, cleaned:15093830, log:(15093830 - 31752192), disk:(11866252-31752192), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:192 ms elapsed: 192 ms 2021-09-13 16:11:08 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: starting... frag 73, data: 14849777, used: 56647680 log:(0 - 56647680) head:0 2021-09-13 16:11:08 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: completed... frag 61, data: 19989722, used: 52493884, disk: 82223017, relocated: 2, retries: 3, skipped: 1, cleaned:56644036, log:(56644036 - 109137920), disk:(26914903-109137920), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:10462 ms elapsed: 10462 ms Mon Sep 13 16:11:13 PDT 2021 RSS = 3267900 2021-09-13T16:11:16.549-07:00 [Info] memstats {"Alloc":530243104, "TotalAlloc":825788327160, "Sys":629883128, "Lookups":0, "Mallocs":26939391,"Frees":26915230, "HeapAlloc":530243104, "HeapSys":597884928, "HeapIdle":65093632, "HeapInuse":532791296,"HeapReleased":26968064, "HeapObjects":24161,"GCSys":20158464, "LastGC":1631574676546991270,"PauseTotalNs":22842611792, "PauseNs":[18086102, 31132194, 25359723, 13404761, 17863727, 17198909, 22195439, 24128114, 21502726, 32467595, 36097004, 39407296, 32120313, 21351245, 22700789, 1739404, 36916665, 31026955, 9633092, 28050806, 26310045, 21417385, 48652990, 18379663, 23295957, 19602001, 28997832, 33019598, 19324096, 19557760, 30500635, 14429456, 5849868, 29642172, 24717389, 5947333, 1985676, 2297461, 44511977, 875673, 1832986, 1865618, 899323, 2086495, 1000837, 2142041, 1263228, 875871, 6286889, 8090743, 13252128, 14455113, 10611390, 15891682, 17984639, 20769371, 292795769, 34923426, 47002211, 25422021, 28493390, 37867497, 34032609, 51143687, 53032842, 36650367, 52099274, 24325433, 45991576, 69230862, 18306055, 185028950, 37260890, 42703876, 30867215, 29760568, 26205960, 14038004, 19896334, 1067539, 39642544, 36643393, 27680022, 768409, 9277956, 76484089, 20117500, 52630889, 109202039, 163042450, 2158764, 72123584], "NumGC":7255} Allocated: 3710179904, active: 3710271488, metadata: 18957520 (n_thp 0), resident: 3767033856, mapped: 3784937472, retained: 1564897280 2021-09-13 16:11:21 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: starting... frag 99, data: 8192, used: 31133696 log:(0 - 31133696) head:0 2021-09-13 16:11:21 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: completed... frag 83, data: 17168228, used: 102952661, disk: 114300993, relocated: 0, retries: 1, skipped: 1, cleaned:31125803, log:(31125803 - 134078464), disk:(19777471-134078464), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:23048 ms elapsed: 23048 ms 2021-09-13T16:11:46.776-07:00 [Info] memstats {"Alloc":404924752, "TotalAlloc":870868881392, "Sys":768430328, "Lookups":0, "Mallocs":29414666,"Frees":29389931, "HeapAlloc":404924752, "HeapSys":733151232, "HeapIdle":325853184, "HeapInuse":407298048,"HeapReleased":203071488, "HeapObjects":24735,"GCSys":24365056, "LastGC":1631574706675957222,"PauseTotalNs":26004315432, "PauseNs":[39254146, 52956385, 37299030, 73490793, 4369071, 42794338, 1648023, 1888781, 134051647, 21337461, 515362, 108756699, 54538814, 41982983, 42672122, 11552337, 48601579, 3045152, 2048656, 55088122, 2364848, 3081146, 1079743, 123196715, 1086230, 1656692, 2104537, 647443, 118187281, 1445568, 976694, 162507130, 81479607, 175086092, 918496, 124579370, 148617726, 132346126, 99307048, 2059603, 1858020, 2850743, 1524891, 1710293, 1612530, 2247821, 1453367, 1527313, 3397726, 962488, 128713149, 6759513, 1283927, 2086509, 1834599, 13654279, 31324085, 1572082, 2230289, 1459008, 1502918, 2455134, 1859596, 1602661, 4224708, 4700736, 1577296, 1384945, 1675684, 1379310, 3273552, 4342620, 1832082, 4233553, 8819517, 8839109, 744729, 1462566, 2301545, 76997192, 473616, 15733990, 1054553, 3840157, 1432740, 54377902, 5250599, 1184671, 1832265, 1942965, 12107623, 2156648, 2643449, 26598135, 2512707, 9982066, 11287177, 17583328, 14705581, 2267992, 1331200, 782646, 2248723, 1272295, 16548744, 21054086, 2297771, 10298726, 6405136, 11769534, 3117420, 3264898, 2190202, 3616333, 2907417, 3534804, 5363878, 1524382, 5731740, 6102926, 1918892, 6797828, 4680964, 14064552, 1888348, 16084344, 1953860, 1249191, 933744, 1570971, 1551524, 3900184, 7094923, 3562021, 1657216, 1323945, 1723687, 1552436, 2846118, 1392392, 3877024, 14103692, 1523548, 2361223, 1699859, 5092546, 6444722, 3832379, 1733884, 2090860, 3341214, 1607747, 1456520, 1255722, 2931178, 1087667, 1330211, 1266645, 4374852, 3593827, 1435593, 1973598, 5299058, 4363679, 2351952, 1553081, 1236134, 1730099, 119904587, 1884356, 1249402, 1661641, 66046385, 1549375, 3471415, 1022320, 2353287, 102339263, 67432657, 4547184, 1797805, 2798757, 1912486, 3742893, 2788996, 2165190, 1825980, 1394851, 896450], "NumGC":7444} Allocated: 3877666944, active: 3877793792, metadata: 19105488 (n_thp 0), resident: 3942866944, mapped: 3960623104, retained: 1389211648 2021-09-13 16:12:02 [ss-workload] instance inse% dele% read% 2021-09-13 16:12:02 [ss-workload] 0 15% 15% 69% 2021-09-13 16:12:02 [ss-workload] 1 15% 15% 69% 2021-09-13 16:12:02 [ss-workload] 2 15% 15% 69% 2021-09-13 16:12:02 [ss-workload] 3 15% 15% 69% 2021-09-13 16:12:02 [ss-workload] 4 3% 3% 92% 2021-09-13 16:12:02 [ss-workload] 5 3% 3% 92% 2021-09-13 16:12:02 [ss-workload] 6 3% 3% 92% 2021-09-13 16:12:02 [ss-workload] 7 18% 18% 62% 2021-09-13 16:12:02 [ss-workload] 8 18% 18% 62% 2021-09-13 16:12:02 [ss-workload] 9 18% 18% 62% 2021-09-13 16:12:02 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:12:02 [ss-diag] 0 21312 20380 0 0 277 1 18156796 2 61175212 65604 1.000000 11.100783 1.000000 0:0:0 2021-09-13 16:12:02 [ss-diag] 1 21777 20977 0 0 312 1 20450976 2 52516572 65588 1.000000 7.090984 1.000000 0:0:0 2021-09-13 16:12:02 [ss-diag] 2 19550 18712 0 0 282 1 18484536 2 55010196 65588 1.000000 1.470120 1.000000 0:0:0 2021-09-13 16:12:02 [ss-diag] 3 19243 18455 0 0 307 1 20123236 2 51729132 65588 1.000000 1.470120 1.000000 0:0:0 2021-09-13 16:12:02 [ss-diag] 4 177693 176809 0 0 907 177 59452036 208 71541188 13577836 1.000000 2.582340 1.000000 0:0:0 2021-09-13 16:12:02 [ss-diag] 5 162961 162063 0 0 861 159 56436828 208 72455580 13578028 1.000000 1.470120 1.000000 0:0:0 2021-09-13 16:12:02 [ss-diag] 6 171043 170141 0 0 813 156 53290524 208 72718220 13577692 1.000000 1.470120 1.000000 0:0:0 2021-09-13 16:12:02 [ss-diag] 7 46091 45173 0 0 107 27 7013636 204 73518204 13315292 1.000000 6.619324 1.000000 0:0:0 2021-09-13 16:12:02 [ss-diag] 8 34644 33699 0 0 313 1 20516524 2 62025520 65588 1.000000 1.470120 1.000000 0:0:0 Mon Sep 13 16:12:13 PDT 2021 RSS = 3538480 2021-09-13 16:12:02 [ss-diag] 9 41432 40484 0 0 111 24 7275828 203 75414192 13249848 1.000000 1.470120 1.000000 0:0:0 2021-09-13T16:12:17.197-07:00 [Info] memstats {"Alloc":385267296, "TotalAlloc":920026392952, "Sys":768430328, "Lookups":0, "Mallocs":32109711,"Frees":32085587, "HeapAlloc":385267296, "HeapSys":733544448, "HeapIdle":345866240, "HeapInuse":387678208,"HeapReleased":206946304, "HeapObjects":24124,"GCSys":24365056, "LastGC":1631574737104659056,"PauseTotalNs":28927402400, "PauseNs":[2830173, 1423787, 1278136, 1162215, 1624540, 3358464, 2814136, 3761602, 2130007, 842269, 1383725, 708846, 1396163, 3248878, 3645935, 1063777, 1701674, 1480917, 2384390, 1673694, 156691048, 182089267, 96038403, 1335729, 1663167, 1415263, 1978166, 1473968, 1336614, 1198323, 11114186, 1766996, 1423129, 78566037, 90517573, 1189934, 7411804, 3597931, 1795248, 2136945, 1563125, 3231394, 46425944, 1070217, 1022636, 1381195, 862706, 152244445, 92609674, 112244943, 3961165, 2310121, 1683535, 456158, 2399024, 25338487, 3119506, 1150851, 1359980, 1710302, 4727815, 1619668, 2517916, 2068302, 900320, 2289657, 3746347, 1147485, 99390272, 1361632, 1434023, 1807878, 1539644, 2109918, 1494273, 184469744, 1618703, 1804199, 3356590, 2023140, 2007027, 2815940, 3123386, 2685118, 2893709, 1541146, 4099106, 2386031, 110977127, 1978409, 1351441, 140575872, 2932216, 190783766, 2528234, 105793259, 1577068, 3097185, 1845395, 2090511, 1900831, 2084138, 1292647, 90325169, 101507223, 83326574, 1658631, 1947948, 686922, 3244538, 1343046, 3265792, 1112946, 1385214, 1185948, 905210, 1128941, 751177, 1447475, 92243191, 1618453, 1832022, 1420359, 4640493, 102525463, 1267408, 1602418, 4039622, 2135374, 709184, 1529105, 976507, 1642899, 1719915, 2453965, 1753955, 2676668, 886153, 1951331, 2423259, 1694487, 902944, 1529289, 2249367, 4668431, 3343485, 1624739, 3029638, 2043761, 3783629, 3769204, 899940, 2584755, 1696820, 1571975, 1008416, 2133706, 1785433, 2484388, 2749165, 3906704, 2011617, 1924906, 1621405, 2374355, 602199, 3734809, 2031965, 1786322, 2154854, 1789826, 1781909, 1321720, 2879558, 3707397, 1493634, 1695484, 3276957, 4029444, 1569039, 1919844, 1779430, 3321827, 2500653, 1472770, 930231, 960258, 2714621, 3291506, 3219082, 5743597, 2148726, 5285134, 2393891, 6032823, 766976, 660291, 2321955, 1901331, 1834388, 176917060, 4262501, 1718083, 1752681, 1853505, 1781688, 1323599, 1843391, 2910282, 3072443, 1348672], "NumGC":7655} Allocated: 3919866752, active: 3919962112, metadata: 19184848 (n_thp 0), resident: 3982774272, mapped: 4000452608, retained: 1349382144 2021-09-13 16:12:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: starting... frag 99, data: 49152, used: 19042270 log:(28496642 - 48549888) head:28496642 2021-09-13 16:12:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: completed... frag 80, data: 23325520, used: 119422566, disk: 131006382, relocated: 0, retries: 0, skipped: 82136, cleaned:383626904, log:(412123546 - 531546112), disk:(400539730-531546112), activeFragRatio:30, activeMaxFragRatio:80, run:277 duration:1840 ms elapsed: 299314 ms 2021-09-13T16:12:47.507-07:00 [Info] memstats {"Alloc":468444808, "TotalAlloc":972962456104, "Sys":768430328, "Lookups":0, "Mallocs":35012155,"Frees":34986896, "HeapAlloc":468444808, "HeapSys":733020160, "HeapIdle":262144000, "HeapInuse":470876160,"HeapReleased":173948928, "HeapObjects":25259,"GCSys":24627200, "LastGC":1631574767482233320,"PauseTotalNs":30337528548, "PauseNs":[3341394, 5338886, 1898046, 2266517, 1672562, 3377993, 2322552, 1362167, 3261802, 1435305, 906398, 1520756, 1976504, 1545827, 691517, 4146155, 1650078, 1677170, 1404107, 4707019, 3034311, 216560524, 1590207, 2675640, 1183510, 2275326, 1319723, 1775809, 3303178, 2103044, 1077703, 873287, 1418519, 598814, 888380, 1005060, 502373, 1963103, 1319686, 1264596, 1436567, 1657065, 1669996, 2689446, 932896, 1650952, 976112, 2137191, 1510211, 1732311, 3094919, 1789220, 1439677, 1064217, 1561136, 4165126, 2184549, 5567605, 1231133, 3703407, 1284613, 1979512, 1846215, 1671965, 117863904, 952720, 125245941, 82062682, 1373995, 2406888, 907762, 1246191, 1177756, 1758833, 2621771, 1697147, 1174054, 1377975, 3179495, 1532287, 1238961, 1013822, 1142791, 597647, 1735328, 1597518, 1030933, 1577070, 1197736, 1922765, 1637059, 1086906, 153841482, 815796, 1913399, 1419699, 1326680, 606828, 2915498, 2065930, 1740839, 2343836, 1696139, 665747, 4919839, 2746762, 855390, 1364359, 1844324, 1600600, 1565311, 2196990, 1786173, 4956450, 1056556, 2586369, 1579399, 969708, 1658281, 3950301, 3140965, 3661698, 4341009, 1059419, 2451410, 1785328, 1774950, 6810091, 1220616, 3065119, 1932896, 4447792, 1409341, 1580072, 2304965, 2928946, 1620660, 2766413, 6607170, 6747613, 1674501, 1758459, 1614196, 1823752, 7064138, 2089692, 1593837, 2221141, 1949446, 1270703, 5661615, 3162264, 988181, 70575361, 1192910, 1380019, 5491000, 1844051, 5538353, 1898107, 944747, 2430630, 2731085, 1098383, 1784700, 1099170, 1093039, 1572162, 5654354, 2338085, 1737108, 1988760, 1611537, 2822663, 2491703, 1627279, 3198696, 874315, 1903052, 1269733, 1360502, 2964273, 3097128, 1240783, 3969135, 3154700, 2054602, 175704732, 736077, 1330558, 1612263, 3847857, 3611636, 786018, 2335608, 1570194, 1178756, 2259994, 3159634, 1812023, 1045936, 2649558, 2825750, 1972892, 1904296, 2362908, 3018583, 3438257, 2589489, 1540275, 1812115, 1188006, 1636931, 2983531, 1473485, 1732185, 1883788, 2572339, 501521, 2881595, 5429879, 1179007, 1993148, 1353459, 4096881, 1996250], "NumGC":7881} Allocated: 3929985728, active: 3930112000, metadata: 19249616 (n_thp 0), resident: 4033122304, mapped: 4050731008, retained: 1299103744 Mon Sep 13 16:13:13 PDT 2021 RSS = 3579388 2021-09-13 16:13:14 [ss-workload] instance inse% dele% read% 2021-09-13 16:13:14 [ss-workload] 0 15% 15% 69% 2021-09-13 16:13:14 [ss-workload] 1 15% 15% 69% 2021-09-13T16:13:17.813-07:00 [Info] memstats {"Alloc":394490144, "TotalAlloc":1027934877280, "Sys":768430328, "Lookups":0, "Mallocs":37985670,"Frees":37958018, "HeapAlloc":394490144, "HeapSys":733282304, "HeapIdle":336601088, "HeapInuse":396681216,"HeapReleased":209149952, "HeapObjects":27652,"GCSys":24463360, "LastGC":1631574797730284140,"PauseTotalNs":32226199487, "PauseNs":[2223317, 1103837, 1967835, 5510977, 2911960, 64430363, 137661562, 1235234, 843593, 1098182, 1179973, 1655272, 3313372, 1426130, 1665211, 5445784, 970476, 1134010, 818197, 1269355, 951832, 586361, 805744, 1397156, 1372544, 1264069, 176001061, 1865647, 3431704, 1538962, 2743436, 1359379, 140045441, 1617173, 1792417, 1793104, 2476510, 3586536, 2226906, 2180484, 5757367, 960006, 4866948, 1316178, 2964161, 1627166, 1764482, 2232914, 1157579, 1594266, 1821479, 4066981, 3878240, 3505072, 1695255, 983402, 1640185, 1501896, 1637507, 2129711, 1948454, 1928959, 1597948, 1522161, 2476750, 1621073, 3607442, 1823280, 1812046, 1260525, 1846383, 1591227, 3270694, 1402228, 1666446, 972705, 1688079, 1431828, 3642036, 2286542, 4002830, 1246516, 1476206, 1792215, 1152831, 2847607, 3075441, 125613128, 1489020, 992020, 1039978, 1116626, 1718490, 1555738, 1462862, 2021890, 1501894, 3087067, 5515773, 1529292, 1447465, 3358406, 3092999, 1507669, 1540420, 3284572, 948273, 2610254, 698295, 1505654, 2851322, 1448231, 1751605, 1420444, 2231816, 3390796, 1270448, 1526516, 1570379, 1220176, 1733702, 1186349, 4135078, 2334850, 667492, 1255552, 2011551, 1381993, 4070257, 195585064, 1628258, 3045539, 1425255, 4444271, 3430772, 1010229, 1685775, 2085419, 2531383, 3295473, 2153160, 1572585, 2242472, 1870714, 3568104, 1117034, 1609153, 1261204, 3157494, 1778159, 1734107, 1179487, 2400969, 862287, 1498950, 1179584, 2980063, 1143564, 463408, 899149, 2689745, 1406626, 1071458, 2028273, 675920, 954842, 998855, 3070229, 1518382, 54349342, 148295029, 3948160, 1949867, 1837011, 1472287, 1730873, 1804159, 2479086, 1798363, 1704262, 3133292, 3259783, 42943769, 6418020, 994936, 2268328, 1485496, 1700161, 2216053, 1425195, 901827, 1604383, 2745821, 848245, 3537398, 1982064, 156422414, 1787879, 1662086, 2789644, 1434780, 208869673, 1488952, 1563716, 2134748, 3978300, 1303131, 1951676, 2031890, 1165223, 1520674, 1439475, 1057290, 1266441, 1628338, 2584712, 1612684, 670678, 1453698, 1898743, 1589571, 1382763, 1523982, 1267006, 505938, 3985730, 2610315, 822787, 2561277, 2923275, 2606130], "NumGC":8112} Allocated: 3963011392, active: 3963125760, metadata: 19254608 (n_thp 0), resident: 4043235328, mapped: 4076666880, retained: 1282785280 2021-09-13 16:13:14 [ss-workload] 2 15% 15% 69% 2021-09-13 16:13:14 [ss-workload] 3 14% 14% 70% 2021-09-13 16:13:14 [ss-workload] 4 3% 3% 92% 2021-09-13 16:13:14 [ss-workload] 5 3% 3% 92% 2021-09-13 16:13:14 [ss-workload] 6 3% 3% 92% 2021-09-13 16:13:14 [ss-workload] 7 18% 18% 62% 2021-09-13 16:13:14 [ss-workload] 8 18% 18% 62% 2021-09-13 16:13:14 [ss-workload] 9 18% 18% 62% 2021-09-13 16:13:14 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:13:14 [ss-diag] 0 24345 23497 0 0 323 1 21172004 2 55663580 65604 1.000000 1.541360 1.000000 0:0:0 2021-09-13 16:13:14 [ss-diag] 1 24037 23035 0 0 336 1 22024128 2 65763556 65588 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:13:14 [ss-diag] 2 22251 21259 0 0 313 1 20516524 2 65108668 65588 1.000000 0.737863 1.000000 0:0:0 2021-09-13 16:13:24 [gsi-test] [] QuotaTuner: 3913788948 -> 3522410053 2021-09-13 16:13:14 [ss-diag] 3 21940 21079 0 0 312 1 20450976 2 56516608 65588 1.000000 0.737863 1.000000 0:0:0 2021-09-13 16:13:14 [ss-diag] 4 268180 267320 0 0 1281 256 83966988 208 69966148 13577836 1.000000 1.653447 1.000000 0:0:0 2021-09-13 16:13:14 [ss-diag] 5 252111 251215 0 0 1308 235 85736784 208 72325556 13578028 1.000000 0.737863 1.000000 0:0:0 2021-09-13 16:13:14 [ss-diag] 6 248705 247785 0 0 1258 221 82459384 208 73899412 13577692 1.000000 0.737863 1.000000 0:0:0 2021-09-13 16:13:14 [ss-diag] 7 53197 52257 0 0 105 25 6882540 208 75222372 13577644 1.000000 2.419496 1.000000 0:0:0 2021-09-13 16:13:14 [ss-diag] 8 39361 38529 0 0 302 1 19795496 2 54614012 65588 1.000000 0.737863 1.000000 0:0:0 2021-09-13 16:13:14 [ss-diag] 9 48461 47481 0 0 127 29 8324596 208 77838564 13577804 1.000000 0.737863 1.000000 0:0:0 2021-09-13T16:13:48.229-07:00 [Info] memstats {"Alloc":417379048, "TotalAlloc":1081750002592, "Sys":768430328, "Lookups":0, "Mallocs":40894014,"Frees":40868267, "HeapAlloc":417379048, "HeapSys":733380608, "HeapIdle":313638912, "HeapInuse":419741696,"HeapReleased":177487872, "HeapObjects":25747,"GCSys":24528896, "LastGC":1631574828156257259,"PauseTotalNs":33767214061, "PauseNs":[2260128, 1446304, 1535453, 1770526, 1701200, 3859210, 1471263, 1521440, 1751075, 1359439, 1048822, 1363978, 2066135, 1723926, 2923462, 2746458, 1837420, 765460, 3062594, 869911, 1336018, 3699729, 1257911, 4730563, 1723001, 1634098, 3394489, 1233962, 2123034, 3069510, 4298280, 677474, 3788729, 2592477, 4361312, 2160489, 1147858, 1933930, 1419554, 208059686, 2080925, 1676650, 1410871, 2510491, 2190357, 1671088, 3228550, 1068995, 3788091, 2948225, 774382, 883787, 5337801, 1265846, 5529834, 1444731, 5658886, 908111, 1433343, 1369417, 1472728, 3681551, 2264171, 3321406, 2912902, 940815, 1482704, 104190972, 874823, 1382731, 1249910, 5817691, 1588487, 1846789, 1277780, 1982700, 1772274, 3029354, 3659011, 1021014, 3815053, 2025284, 3009025, 3583409, 1384636, 4881862, 3838816, 1620936, 457655, 1789694, 3523144, 1308060, 1564830, 2497560, 1465538, 1480141, 2832903, 2283815, 1887239, 2259419, 497818, 2022680, 1711245, 3817214, 4493427, 1694328, 3580312, 3030023, 2479374, 3299986, 229783495, 158919673, 1042576, 1605811, 1564795, 2019021, 3281371, 1267252, 901897, 1369064, 1219813, 2618359, 1739043, 2105641, 169750173, 3063877, 1775116, 2343228, 1563725, 3142820, 1730732, 2395253, 1508157, 898730, 1870658, 1532629, 1287642, 796288, 4680840, 3337276, 1213835, 2102531, 1560412, 676228, 1751815, 1900208, 1212536, 558341, 1233043, 4056911, 1295799, 1268781, 1936608, 1685149, 1886682, 1027568, 1183000, 1554215, 1548465, 1186248, 6367721, 3067327, 1680809, 2605531, 1039853, 1731425, 1229288, 1251820, 3362547, 5437330, 1852978, 1589758, 4070173, 1249038, 2074461, 1284679, 2905185, 1746437, 1764362, 1755923, 2358485, 1471538, 1019137, 3259528, 2164524, 2046502, 1024655, 5508573, 4739772, 1669330, 1713780, 2631545, 642681, 1488567, 1080278, 1418418, 141008504, 2035467, 2494543, 773909, 6062743, 1783786, 2599993, 1164998, 2861055, 1027745, 3226447, 2752906, 6209179, 1254483, 544310, 756908, 1274549, 2516094, 876073, 4962150, 5089293, 2358179, 2395622, 1783217, 1747957, 1624358, 2031908, 4935615, 3231893, 4795786, 40718255, 1047433], "NumGC":8340} Allocated: 3955248704, active: 3955372032, metadata: 19254608 (n_thp 0), resident: 4059385856, mapped: 4076990464, retained: 1390284800 Mon Sep 13 16:14:13 PDT 2021 RSS = 2778932 2021-09-13T16:14:18.389-07:00 [Info] memstats {"Alloc":249936240, "TotalAlloc":1135852429112, "Sys":768430328, "Lookups":0, "Mallocs":43779672,"Frees":43761557, "HeapAlloc":249936240, "HeapSys":733413376, "HeapIdle":480690176, "HeapInuse":252723200,"HeapReleased":196116480, "HeapObjects":18115,"GCSys":24463360, "LastGC":1631574858373942276,"PauseTotalNs":35601848343, "PauseNs":[1370206, 1751461, 3145456, 686939, 1773291, 3728580, 2183365, 1013437, 873653, 1935552, 1597608, 3897621, 2164881, 1511615, 1650720, 1347965, 791573, 1555596, 2765881, 849785, 1467870, 1584392, 364660, 1182885, 3310539, 1868451, 1131778, 4605420, 1430225, 1900374, 574030, 3146919, 931838, 1513591, 176150388, 84752011, 1761314, 3004653, 3920272, 1814556, 1233587, 934401, 1771824, 4717305, 1571390, 1214707, 1553274, 1172203, 1836816, 1495389, 160027334, 4771373, 1477966, 2186679, 937312, 4072881, 1576101, 182050506, 1298784, 985862, 1150670, 3680007, 1753042, 1643627, 1192383, 1521858, 969673, 2025595, 4598848, 1965507, 1623079, 1624258, 3638320, 688388, 1163587, 1053823, 2300277, 1743370, 1735021, 2294289, 2516724, 2079607, 1778327, 2507942, 1424499, 3546327, 3752464, 823684, 1464654, 1636855, 717797, 1882463, 1467666, 2810387, 2358787, 2102062, 418284, 4973460, 4671279, 3013362, 1369889, 1478850, 832700, 1374115, 1596763, 1337170, 1979643, 3033865, 1884771, 1565942, 1695468, 1450583, 2727666, 7820750, 1823649, 1173088, 3574159, 3050724, 2914996, 3568201, 110722996, 1917670, 1451417, 4798396, 2668782, 1774446, 1465697, 1538819, 1242590, 2390976, 3159185, 1553884, 1306514, 1702468, 1385137, 2674789, 1337457, 2726022, 2076962, 1845900, 3715350, 1055429, 1676392, 1135669, 625444, 562643, 1142522, 3668506, 2063261, 1751585, 1338993, 134672277, 1284145, 1269553, 1687259, 1612107, 388935, 1996522, 905298, 1994794, 1301708, 3074767, 1563107, 931585, 1313726, 845248, 1478446, 2187305, 1227018, 3676051, 1661613, 2206783, 1787932, 1389068, 2962513, 2154304, 1600928, 1266323, 1425940, 1524214, 1434372, 638001, 1074148, 3266568, 3365458, 887167, 2250449, 4159068, 1214714, 1012954, 101853737, 152551560, 45514127, 1978361, 2051504, 4161363, 1405748, 109361517, 2473956, 4223928, 718293, 1734055, 1561728, 1660835, 1812598, 1175880, 2174858, 1513866, 1386728, 2046146, 1456633, 1190406, 1421899, 1881465, 1267323, 2413514, 3438674, 1799859, 1568602, 1448620, 1237067, 1060016, 1851329, 1358524, 1881490, 2024952, 1480909, 4060552, 1276360, 139981891, 908660, 1523043, 1639590, 3315585, 2155973], "NumGC":8575} Allocated: 2667849472, active: 2667974656, metadata: 19277648 (n_thp 0), resident: 2731278336, mapped: 2748862464, retained: 2718412800 2021-09-13 16:14:30 [ss-workload] instance inse% dele% read% 2021-09-13 16:14:30 [ss-workload] 0 15% 15% 69% 2021-09-13 16:14:30 [ss-workload] 1 15% 15% 69% 2021-09-13 16:14:30 [ss-workload] 2 15% 15% 69% 2021-09-13 16:14:30 [ss-workload] 3 15% 15% 69% 2021-09-13 16:14:30 [ss-workload] 4 3% 3% 92% 2021-09-13 16:14:30 [ss-workload] 5 3% 3% 92% 2021-09-13 16:14:30 [ss-workload] 6 3% 3% 92% 2021-09-13 16:14:30 [ss-workload] 7 18% 18% 62% 2021-09-13 16:14:30 [ss-workload] 8 18% 18% 62% 2021-09-13 16:14:30 [ss-workload] 9 18% 18% 62% 2021-09-13 16:14:30 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:14:30 [ss-diag] 0 27012 26134 0 0 341 1 22351868 2 57630484 65604 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:14:30 [ss-diag] 1 26944 26198 0 0 327 1 21434196 2 48973796 65588 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:14:30 [ss-diag] 2 25323 24411 0 0 310 1 20319880 2 59861468 65588 1.000000 0.775677 1.000000 0:0:0 2021-09-13 16:14:30 [ss-diag] 3 24579 23677 0 0 313 1 20516524 2 59205780 65588 1.000000 0.775677 1.000000 0:0:0 2021-09-13 16:14:30 [ss-diag] 4 358481 357569 0 0 1648 314 108023104 208 73375796 13577836 1.000000 1.623766 1.000000 0:0:0 2021-09-13 16:14:30 [ss-diag] 5 337663 336735 0 0 1198 215 78526504 208 74422324 13578028 1.000000 0.775677 1.000000 0:0:0 2021-09-13 16:14:30 [ss-diag] 6 339236 338276 0 0 1762 307 115495576 208 76520340 13577692 1.000000 0.775677 1.000000 0:0:0 2021-09-13 16:14:30 [ss-diag] 7 62177 61171 0 0 154 32 10094392 208 79547468 13577644 1.000000 2.275657 1.000000 0:0:0 2021-09-13 16:14:30 [ss-diag] 8 44113 43333 0 0 326 1 21368648 2 51202444 65588 1.000000 0.775677 1.000000 0:0:0 2021-09-13 16:14:30 [ss-diag] 9 54208 53225 0 0 119 29 7800212 208 78033952 13577804 1.000000 0.775677 1.000000 0:0:0 2021-09-13T16:14:48.692-07:00 [Info] memstats {"Alloc":423171256, "TotalAlloc":1190669495352, "Sys":768430328, "Lookups":0, "Mallocs":46731839,"Frees":46704669, "HeapAlloc":423171256, "HeapSys":733151232, "HeapIdle":307568640, "HeapInuse":425582592,"HeapReleased":202088448, "HeapObjects":27170,"GCSys":24463360, "LastGC":1631574888627499250,"PauseTotalNs":36744408973, "PauseNs":[959837, 1296080, 1418799, 1922407, 1337897, 1520402, 121529444, 1571845, 1830316, 2160132, 1531317, 2119494, 4307940, 1329785, 2892002, 2979908, 2072324, 1752527, 2065933, 3061553, 3433190, 1710549, 1649684, 1436099, 4797380, 1458609, 2117562, 1974920, 1155176, 1935787, 3633518, 2462952, 914091, 1350937, 2520606, 1790695, 2701086, 1833388, 1390441, 1090394, 7033140, 3731035, 1879311, 2279727, 1689688, 1787742, 1873715, 1165471, 1621964, 2880651, 2655205, 1402613, 1787077, 2279409, 3369380, 1015269, 1002859, 591009, 966984, 1335606, 1055517, 3269529, 4771253, 1631617, 1782223, 5788833, 1046510, 1786013, 1638067, 1678623, 1241264, 2297829, 2608273, 2116522, 2086779, 1180696, 3517156, 2450939, 741426, 2161783, 3887411, 4302798, 3379192, 1407769, 1519933, 1974807, 2012872, 1145875, 3614715, 1572005, 1612220, 1750540, 2119721, 1704174, 3073064, 1800559, 1395718, 4123560, 3994614, 1583312, 75176621, 5971856, 2194202, 1554099, 490369, 1124705, 1993699, 1921080, 850878, 1541124, 4887748, 1017073, 2695995, 1322806, 1741172, 973718, 3699839, 1626402, 1789936, 882137, 2146571, 2874928, 1738874, 1727651, 1558471, 1937580, 866127, 1917386, 4058359, 2772105, 2985897, 3110039, 1209627, 1297554, 2212469, 3688050, 2085620, 1754323, 2134178, 932216, 2202617, 1365075, 1599093, 1397554, 1788547, 2272894, 1301320, 1578464, 1712720, 1739806, 1479228, 3148657, 3381841, 2188123, 1491347, 3143709, 2096599, 1502148, 2233867, 2977170, 1398057, 1584300, 2981506, 820843, 756770, 221747699, 1662382, 1872858, 1473763, 2137652, 2378794, 1596374, 3579226, 953621, 3440666, 1351173, 1615162, 1688460, 995790, 1623623, 198353795, 1885578, 1666234, 3641589, 3189908, 1467092, 52900150, 3173202, 1963168, 1545812, 2450673, 621336, 1773967, 1562643, 4899005, 3057811, 1472209, 1430779, 2065566, 1266012, 714398, 2651341, 878491, 2438289, 1015644, 661087, 1028423, 1315301, 713441, 1702640, 2525803, 1096170, 1718819, 1666667, 2107576, 2539090, 1870125, 4335692, 946584, 1014240, 1732782, 2550351, 1381016, 1491176, 971553, 2545883, 1166787, 1047941, 1302506, 1390908, 4820435, 1283949, 1575123, 1519103, 2191394, 3349819], "NumGC":8811} Allocated: 2676121536, active: 2676232192, metadata: 19283408 (n_thp 0), resident: 2777849856, mapped: 2795425792, retained: 2671849472 Mon Sep 13 16:15:13 PDT 2021 RSS = 2971932 2021-09-13T16:15:19.270-07:00 [Info] memstats {"Alloc":391086592, "TotalAlloc":1239386139848, "Sys":768430328, "Lookups":0, "Mallocs":49364813,"Frees":49341657, "HeapAlloc":391086592, "HeapSys":733446144, "HeapIdle":339820544, "HeapInuse":393625600,"HeapReleased":211206144, "HeapObjects":23156,"GCSys":24430592, "LastGC":1631574919249644003,"PauseTotalNs":38805603960, "PauseNs":[1057414, 2791605, 1388842, 1745285, 1395751, 1772691, 4246177, 5835963, 3871155, 6000897, 1063270, 3200918, 6041122, 2922572, 136149843, 1669135, 3053468, 1551930, 1243307, 1105296, 1291247, 3160132, 1658080, 1798679, 2011916, 1506764, 4136228, 1674210, 2213029, 1731998, 2120576, 1517034, 1772050, 2012727, 1900628, 1026647, 1459110, 3848122, 2057814, 1731621, 1523834, 754946, 1114859, 908473, 3460433, 1501672, 1810673, 2500800, 1105015, 2259060, 5725598, 3573614, 2839294, 1923421, 829803, 1560362, 1583439, 174555439, 1897126, 4048396, 819617, 2139464, 1783298, 774805, 1510799, 780695, 1895988, 1425114, 1411278, 1504331, 1861892, 2163533, 2344563, 1411558, 1411020, 1349557, 1308346, 1691026, 895544, 6306544, 1497250, 5022170, 2296953, 567868, 3313979, 587156, 3485144, 4257346, 1348857, 1291462, 1871374, 1777379, 1463084, 1221203, 4758602, 996915, 1304648, 2698826, 3215184, 894182, 727914, 1372966, 1142537, 3008164, 3195200, 4683981, 130579033, 524166, 1998966, 1544123, 1289904, 1348586, 2128815, 1422247, 1278058, 1453476, 1417712, 183318749, 2245528, 1802423, 1398204, 4079798, 1230509, 2898882, 1516004, 2608315, 1069761, 1805881, 1395726, 28040409, 110740523, 7226779, 4729051, 25177367, 19078826, 6974192, 18123992, 21029048, 178176020, 20695246, 26037643, 11038057, 30480821, 31728600, 20685715, 9421842, 38631271, 17902804, 8740163, 30263196, 18273942, 4463802, 7334250, 15413461, 8722025, 16687899, 4306820, 3860615, 23160906, 4477669, 24663570, 5884690, 3232448, 8739346, 16263881, 12261870, 16911991, 2197870, 17424586, 16234949, 18469884, 29762218, 1648085, 833846, 1418470, 1871559, 1927783, 1315945, 1833965, 2734000, 1924501, 6018971, 1043994, 1520586, 1992901, 1417383, 1627508, 1739700, 2206928, 1750352, 3014980, 2271475, 452790, 161412921, 1429059, 1793743, 2232662, 1942442, 947430, 1545774, 1411864, 1514425, 1608414, 1206795, 2198880, 1831320, 1247238, 1839745, 1623735, 3816869], "NumGC":9021} Allocated: 2802118208, active: 2802233344, metadata: 19283408 (n_thp 0), resident: 2873655296, mapped: 2891231232, retained: 2576044032 2021-09-13 16:15:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 229, "num_active_sctxs": 41, "num_free_sctxs": 188, "num_trim_sctxs": 93, "num_wctxs": 204, "num_free_wctxs": 189, "num_active_wctxs": 15, "num_trim_wctxs": 342, "num_readers": 89, "num_writers": 81, "cpu": 10.20008, "total_core": 16, "threads": 46, "assigned_quota": 3522410053, "current_quota": 3522410053, "memory_resident": 0, "memory_jemalloc": 2902687744, "memory_jemalloc_allocated": 2781130304, "memory_in_use": 2901377024, "memory_stats_size_total": 2469370833, "memory_stats_size_global": 736440176, "memory_stats_size_page": 668220332, "memory_stats_size_index": 68282564, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 162256, "reclaim_pending": 7343936, "buf_memused": 1725524001, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 12, "num_active_smr": 5, "num_pending_smr": 1, "num_pages": 1051, "items_count": 2080, "total_records": 9150, "total_ops": 370570, "compacts": 148483, "purges": 0, "lss_data_size": 188654118, "lss_used_space": 689236973, "lss_fragmentation": 72, "lss_num_reads": 172252, "lss_read_bs": 43347072772, "lss_blk_read_bs": 44198408192, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 44002729984, "bytes_incoming": 24290122360, "write_amp_avg": 0.86, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 170 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9150, "resident_ratio": 100.00000, "cleaner_num_reads": 172252, "cleaner_read_bs": 43347072772, "cleaner_blk_read_bs": 44198408192, "recovery_num_lssctx": 5, "recovery_num_sctxs": 28, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 8, "recovery_num_trim_sctxs": 8, "recovery_buf_memused": 3145728, "recovery_data_size": 48142564, "recovery_used_space": 196721964, "recovery_lss_blk_read_bs": 18882883584, "recovery_lss_read_bs": 18193908345, "recovery_lss__num_reads": 142099, "recovery_bytes_written": 18421755904, "recovery_lss_frag": 75, "recovery_cleaner_num_reads": 142099, "recovery_cleaner_read_bs": 18193908345, "recovery_cleaner_blk_read_bs": 18882883584, "mvcc_purge_ratio": 4.39904 }2021-09-13 16:15:40 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: starting... frag 99, data: 42151027, used: 8434234542 log:(4186045266 - 12620279808) head:4186045266 2021-09-13 16:15:40 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: completed... frag 70, data: 111128535, used: 381706809, disk: 428237808, relocated: 50145, retries: 14950, skipped: 5461, cleaned:13133042293, log:(17319087559 - 17700794368), disk:(17272556560-17700794368), activeFragRatio:30, activeMaxFragRatio:80, run:12 duration:281806 ms elapsed: 281807 ms 2021-09-13 16:15:42 [ss-workload] instance inse% dele% read% 2021-09-13 16:15:42 [ss-workload] 0 14% 14% 70% 2021-09-13 16:15:42 [ss-workload] 1 14% 14% 70% 2021-09-13 16:15:42 [ss-workload] 2 15% 15% 69% 2021-09-13 16:15:42 [ss-workload] 3 14% 14% 70% 2021-09-13 16:15:42 [ss-workload] 4 3% 3% 92% 2021-09-13 16:15:42 [ss-workload] 5 3% 3% 92% 2021-09-13 16:15:42 [ss-workload] 6 3% 3% 92% 2021-09-13 16:15:42 [ss-workload] 7 18% 18% 62% 2021-09-13 16:15:42 [ss-workload] 8 18% 18% 62% 2021-09-13 16:15:42 [ss-workload] 9 18% 18% 62% 2021-09-13 16:15:42 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13T16:15:49.634-07:00 [Info] memstats {"Alloc":422296584, "TotalAlloc":1293745885368, "Sys":768430328, "Lookups":0, "Mallocs":52282484,"Frees":52254866, "HeapAlloc":422296584, "HeapSys":733249536, "HeapIdle":308600832, "HeapInuse":424648704,"HeapReleased":195756032, "HeapObjects":27618,"GCSys":24496128, "LastGC":1631574949552642097,"PauseTotalNs":40354208679, "PauseNs":[3786328, 1310624, 1498070, 1355916, 1135580, 2855311, 1526859, 1988900, 836972, 781437, 923040, 1573125, 1591383, 1407332, 5418857, 841181, 1518723, 4768298, 62832575, 4592732, 1072357, 1481574, 1986619, 1178082, 1102008, 967433, 1301057, 864686, 2090923, 1526088, 1093720, 1381140, 1509893, 2384441, 1212561, 1358431, 5958079, 1163677, 2851572, 3158398, 1429349, 1145380, 2893844, 2995892, 3269553, 2028550, 1007096, 1182833, 2145676, 158326611, 2877365, 1262111, 1969105, 1113948, 1916298, 1408022, 1395611, 2617926, 1192867, 1539590, 1351983, 912853, 1278769, 1067550, 994823, 2621464, 1608749, 2754114, 1153451, 538630, 1774475, 4776613, 1390721, 5761592, 1128368, 2861945, 128748768, 1515229, 1546453, 3916138, 1579841, 2026444, 1605528, 1105917, 2083098, 182737326, 33712781, 1576578, 2282215, 3578819, 893327, 3215934, 56187234, 2453952, 1931294, 1501544, 3592781, 1794616, 62409516, 805425, 1447864, 1544319, 2057361, 1536990, 1366881, 3875107, 1293006, 1067179, 1080343, 703758, 1818071, 1448186, 3512455, 3930294, 991635, 895758, 747123, 3714431, 2326495, 1178883, 1349745, 2531700, 868313, 823046, 3179453, 2695567, 1526522, 1456802, 1718980, 1040946, 2873092, 2862037, 1499781, 2660484, 1914570, 4735047, 1441536, 1012841, 704065, 5636876, 1797504, 1387567, 4809769, 2214134, 1960964, 3401019, 1268284, 1660144, 950010, 1476355, 2117758, 827192, 910068, 1521578, 1375418, 1490984, 1595792, 3560771, 1513253, 1249814, 1516322, 3321524, 192637767, 3253503, 1811372, 4220018, 3608485, 1062195, 1544935, 4073426, 1303364, 2745378, 1280360, 3168871, 1743136, 1780081, 5426246, 900067, 1612496, 2264666, 1374598, 3446363, 1735632, 1171902, 1382012, 1328880, 1754605, 2134714, 1510066, 1858510, 2035378, 792406, 2585939, 2597489, 2409979, 1636343, 1868544, 2899510, 2006088, 1372543, 1439993, 1462115, 1253340, 679668, 1930238, 5597960, 1695388, 1369680, 1508047, 1760659, 1376481, 1184943, 1208166, 1647551, 1049907, 2698404, 1030363, 1080469, 3469660, 4033482, 106268328, 2304185, 3930833, 1043300, 127380917, 1704247, 1471579, 1408710, 1154752, 1360876, 1723609, 1364769], "NumGC":9253} Allocated: 2783349952, active: 2783461376, metadata: 19297232 (n_thp 0), resident: 2851815424, mapped: 2869379072, retained: 2597896192 2021-09-13 16:15:42 [ss-diag] 0 29280 28284 0 0 307 1 20123236 2 65370188 65604 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:15:42 [ss-diag] 1 29134 28302 0 0 335 1 21958580 2 54613148 65588 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:15:42 [ss-diag] 2 28012 27086 0 0 322 1 21106456 2 60778612 65588 1.000000 0.800919 1.000000 0:0:0 2021-09-13 16:15:42 [ss-diag] 3 27230 26310 0 0 335 1 21958580 2 60385468 65588 1.000000 0.800919 1.000000 0:0:0 2021-09-13 16:15:42 [ss-diag] 4 438934 438088 0 0 1360 252 89145280 208 69050220 13577836 1.000000 1.626121 1.000000 0:0:0 2021-09-13 16:15:42 [ss-diag] 5 415412 414440 0 0 1770 324 116019960 208 77307588 13578028 1.000000 0.800919 1.000000 0:0:0 2021-09-13 16:15:42 [ss-diag] 6 414696 413758 0 0 1287 232 84360276 208 75078140 13577692 1.000000 0.800919 1.000000 0:0:0 2021-09-13 16:15:42 [ss-diag] 7 69998 69054 0 0 133 33 8717884 208 75482772 13577644 1.000000 2.210839 1.000000 0:0:0 2021-09-13 16:15:42 [ss-diag] 8 48691 47791 0 0 311 1 20385428 3 59139344 131192 1.000000 0.800919 1.000000 0:0:0 2021-09-13 16:15:42 [ss-diag] 9 63642 62634 0 0 201 33 13175148 208 79671636 13577804 1.000000 0.800919 1.000000 0:0:0 2021-09-13 16:15:59 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: starting... frag 84, data: 9605009, used: 62509746 log:(66919758 - 129429504) head:30338645 2021-09-13 16:15:59 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: completed... frag 30, data: 33196850, used: 47628174, disk: 51883690, relocated: 616378, retries: 463, skipped: 54758, cleaned:1294838531, log:(1361758289 - 1410174976), disk:(1358291286-1410174976), activeFragRatio:30, activeMaxFragRatio:80, run:277 duration:72967 ms elapsed: 294099 ms 2021-09-13 16:16:00 [gsi-test] [] Stop and verify... 2021-09-13 16:16:01 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper stopped 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 62, data: 9187532, used: 24772144 log:(14746064 - 40505344) head:14746064 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 80, data: 8400740, used: 42659451, disk: 44109404, relocated: 7914573, retries: 28842, skipped: 2060985, cleaned:10879692826, log:(10894438890 - 10937495552), disk:(10893386148-10937495552), activeFragRatio:30, activeMaxFragRatio:80, run:243 duration:293868 ms elapsed: 295326 ms 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:16:01 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : All daemons stopped 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper stopped 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:16:01 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : All daemons stopped 2021-09-13 16:16:01 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper stopped 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:16:01 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:16:02 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:16:02 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:16:02 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:16:02 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:16:02 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:16:02 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:16:02 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:16:02 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:16:02 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:16:02 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:16:02 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:16:02 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:16:02 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:16:02 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : All daemons stopped 2021-09-13 16:16:02 [gsi-test] [0] Stop and verify... 2021-09-13 16:16:02 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:16:03 [gsi-test] [0] Stop and verify... OK 2021-09-13 16:16:03 [gsi-test] [1] Stop and verify... 2021-09-13 16:16:04 [gsi-test] [1] Stop and verify... OK 2021-09-13 16:16:04 [gsi-test] [2] Stop and verify... 2021-09-13 16:16:06 [gsi-test] [2] Stop and verify... OK 2021-09-13 16:16:06 [gsi-test] [3] Stop and verify... 2021-09-13 16:16:08 [gsi-test] [3] Stop and verify... OK 2021-09-13 16:16:08 [gsi-test] [4] Stop and verify... 2021-09-13 16:16:08 [gsi-test] [4] Stop and verify... OK 2021-09-13 16:16:08 [gsi-test] [5] Stop and verify... 2021-09-13 16:16:08 [gsi-test] [5] Stop and verify... OK 2021-09-13 16:16:08 [gsi-test] [6] Stop and verify... 2021-09-13 16:16:08 [gsi-test] [6] Stop and verify... OK 2021-09-13 16:16:08 [gsi-test] [7] Stop and verify... 2021-09-13 16:16:09 [gsi-test] [7] Stop and verify... OK 2021-09-13 16:16:09 [gsi-test] [8] Stop and verify... 2021-09-13 16:16:10 [gsi-test] [8] Stop and verify... OK 2021-09-13 16:16:10 [gsi-test] [9] Stop and verify... 2021-09-13 16:16:10 [gsi-test] [] { "memory_quota": 3522410053, "count": 12622, "compacts": 63, "purges": 0, "splits": 1, "merges": 0, "inserts": 12622, "deletes": 0, "compact_conflicts": 151, "split_conflicts": 5, "merge_conflicts": 0, "insert_conflicts": 2447, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 2, "memory_size": 55138460, "memory_size_index": 65604, "allocated": 1980003348, "freed": 1924864888, "reclaimed": 1924864888, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 65604, "freed_index": 0, "reclaimed_index": 0, "num_pages": 2, "items_count": 208, "total_records": 840, "num_rec_allocs": 30186, "num_rec_frees": 29346, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 827346856, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 819200, "est_recovery_size": 1402332, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 37644, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 25022, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 4.03846, "currSn": 894, "gcSn": 892, "gcSnIntervals": "[0 581 893]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 15, "num_trim_wctxs": 26, "num_readers": 8, "num_writers": 7, "page_bytes": 20319880, "page_cnt": 1, "page_itemcnt": 310, "avg_item_size": 65548, "avg_page_size": 20319880, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 65697632, "page_bytes_marshalled":474898298, "page_bytes_compressed":258484620, "compression_ratio": 1.83724, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 7312, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 183668271, "lss_data_size": 16448, "lss_used_space": 15865530, "lss_disk_size": 76853248, "lss_fragmentation": 99, "lss_num_reads": 68, "lss_read_bs": 552189890, "lss_blk_read_bs": 553385984, "bytes_written": 546615296, "bytes_incoming": 827346856, "write_amp": 0.00, "write_amp_avg": 0.66, "lss_gc_num_reads": 68, "lss_gc_reads_bs": 552189890, "lss_blk_gc_reads_bs": 553385984, "lss_gc_status": "frag 0, data: 29904905, used: 13059770, disk: 86284707, relocated: 97, retries: 85, skipped: 20, cleaned:499623963, log:(530749766 - 543809536), disk:(457524829-543809536), activeFragRatio:30, activeMaxFragRatio:80, run:11 duration:22521 ms elapsed: 40896 ms", "lss_head_offset": 530749766, "lss_tail_offset": 543809536, "num_sctxs": 29, "num_free_sctxs": 19, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:16:10 [gsi-test] [] { "memory_quota": 3522410053, "count": 12668, "compacts": 63, "purges": 0, "splits": 1, "merges": 0, "inserts": 12668, "deletes": 0, "compact_conflicts": 126, "split_conflicts": 2, "merge_conflicts": 0, "insert_conflicts": 2247, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 1, "memory_size": 58548844, "memory_size_index": 65588, "allocated": 1950701224, "freed": 1892152380, "reclaimed": 1892152380, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 65588, "freed_index": 0, "reclaimed_index": 0, "num_pages": 2, "items_count": 208, "total_records": 892, "num_rec_allocs": 29739, "num_rec_frees": 28847, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 830362064, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 819200, "est_recovery_size": 1332632, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 37755, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 25087, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 4.28846, "currSn": 900, "gcSn": 895, "gcSnIntervals": "[0 579 896]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 2, "num_free_wctxs": 15, "num_trim_wctxs": 36, "num_readers": 10, "num_writers": 6, "page_bytes": 21958580, "page_cnt": 1, "page_itemcnt": 335, "avg_item_size": 65548, "avg_page_size": 21958580, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 48907632, "page_bytes_marshalled":406598396, "page_bytes_compressed":221317414, "compression_ratio": 1.83717, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 9152, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 91186665, "lss_data_size": 16448, "lss_used_space": 15081641, "lss_disk_size": 59961344, "lss_fragmentation": 99, "lss_num_reads": 57, "lss_read_bs": 400186796, "lss_blk_read_bs": 400297984, "bytes_written": 395505664, "bytes_incoming": 830362064, "write_amp": 0.00, "write_amp_avg": 0.47, "lss_gc_num_reads": 57, "lss_gc_reads_bs": 400186796, "lss_blk_gc_reads_bs": 400297984, "lss_gc_status": "frag 0, data: 25839344, used: 12738729, disk: 75150109, relocated: 89, retries: 76, skipped: 17, cleaned:323779987, log:(380424023 - 393162752), disk:(318012643-393162752), activeFragRatio:30, activeMaxFragRatio:80, run:9 duration:18978 ms elapsed: 19139 ms", "lss_head_offset": 380424023, "lss_tail_offset": 393162752, "num_sctxs": 33, "num_free_sctxs": 23, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:16:11 [gsi-test] [] { "memory_quota": 3522410053, "count": 11944, "compacts": 60, "purges": 0, "splits": 1, "merges": 0, "inserts": 11944, "deletes": 0, "compact_conflicts": 141, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 2153, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 2, "memory_size": 50022884, "memory_size_index": 65588, "allocated": 1897250188, "freed": 1847227304, "reclaimed": 1847227304, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 65588, "freed_index": 0, "reclaimed_index": 0, "num_pages": 2, "items_count": 208, "total_records": 762, "num_rec_allocs": 28923, "num_rec_frees": 28161, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 782905312, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 204800, "est_recovery_size": 139534, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 35681, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 23737, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 3.66346, "currSn": 797, "gcSn": 794, "gcSnIntervals": "[0 466 795]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 16, "num_trim_wctxs": 36, "num_readers": 9, "num_writers": 8, "page_bytes": 19861044, "page_cnt": 1, "page_itemcnt": 303, "avg_item_size": 65548, "avg_page_size": 19861044, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 62943016, "page_bytes_marshalled":718673752, "page_bytes_compressed":391161829, "compression_ratio": 1.83728, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 4552, "num_total_smr": 1, "num_active_smr": 3, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 1099631379, "lss_data_size": 14936734, "lss_used_space": 411348534, "lss_disk_size": 454909952, "lss_fragmentation": 96, "lss_num_reads": 22395, "lss_read_bs": 19099923109, "lss_blk_read_bs": 19531300864, "bytes_written": 19446718464, "bytes_incoming": 16546543832, "write_amp": 0.82, "write_amp_avg": 1.09, "lss_gc_num_reads": 22395, "lss_gc_reads_bs": 19099923109, "lss_blk_gc_reads_bs": 19531300864, "lss_gc_status": "frag 67, data: 98780301, used: 305421960, disk: 343390578, relocated: 17276, retries: 2645, skipped: 1142, cleaned:421389233, log:(17740476792 - 18045898752), disk:(17702508174-18045898752), activeFragRatio:30, activeMaxFragRatio:80, run:2 duration:21737 ms elapsed: 21737 ms", "lss_head_offset": 17740476792, "lss_tail_offset": 18045898752, "num_sctxs": 101, "num_free_sctxs": 90, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:16:12 [gsi-test] [] { "memory_quota": 3522410053, "count": 11826, "compacts": 59, "purges": 0, "splits": 1, "merges": 0, "inserts": 11826, "deletes": 0, "compact_conflicts": 163, "split_conflicts": 2, "merge_conflicts": 0, "insert_conflicts": 2446, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 1, "memory_size": 55925580, "memory_size_index": 65588, "allocated": 1851750484, "freed": 1795824904, "reclaimed": 1795824904, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 65588, "freed_index": 0, "reclaimed_index": 0, "num_pages": 2, "items_count": 208, "total_records": 852, "num_rec_allocs": 28228, "num_rec_frees": 27376, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 775170648, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 204800, "est_recovery_size": 139534, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 35218, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 23392, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 4.09615, "currSn": 791, "gcSn": 782, "gcSnIntervals": "[0 468 783]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 20, "num_trim_wctxs": 33, "num_readers": 13, "num_writers": 5, "page_bytes": 21958580, "page_cnt": 1, "page_itemcnt": 335, "avg_item_size": 65548, "avg_page_size": 21958580, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 65436000, "page_bytes_marshalled":707989312, "page_bytes_compressed":385354612, "compression_ratio": 1.83724, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 7872, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 1099631379, "lss_data_size": 14936734, "lss_used_space": 411348534, "lss_disk_size": 454909952, "lss_fragmentation": 96, "lss_num_reads": 22395, "lss_read_bs": 19099923109, "lss_blk_read_bs": 19531300864, "bytes_written": 19446718464, "bytes_incoming": 16557293704, "write_amp": 0.82, "write_amp_avg": 1.09, "lss_gc_num_reads": 22395, "lss_gc_reads_bs": 19099923109, "lss_blk_gc_reads_bs": 19531300864, "lss_gc_status": "frag 67, data: 98780301, used: 305421960, disk: 343390578, relocated: 17276, retries: 2645, skipped: 1142, cleaned:421389233, log:(17740476792 - 18045898752), disk:(17702508174-18045898752), activeFragRatio:30, activeMaxFragRatio:80, run:2 duration:21737 ms elapsed: 21737 ms", "lss_head_offset": 17740476792, "lss_tail_offset": 18045898752, "num_sctxs": 101, "num_free_sctxs": 91, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:16:12 [gsi-test] [] { "memory_quota": 3522410053, "count": 97534, "compacts": 60740, "purges": 0, "splits": 207, "merges": 0, "inserts": 97534, "deletes": 0, "compact_conflicts": 243, "split_conflicts": 28, "merge_conflicts": 0, "insert_conflicts": 2409, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 2, "memory_size": 75994964, "memory_size_index": 13577836, "allocated": 34927472228, "freed": 34851477264, "reclaimed": 34846887192, "reclaim_pending": 4590072, "reclaim_list_size": 2819852, "reclaim_list_count": 7, "reclaim_threshold": 1, "allocated_index": 13577836, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 952, "num_rec_allocs": 472062, "num_rec_frees": 471110, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 6393158632, "lss_data_size": 365596, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 36559600, "est_recovery_size": 42236102, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 1393063, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 1295529, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 4.57692, "currSn": 15428, "gcSn": 15398, "gcSnIntervals": "[0 9651 15399]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 7, "num_free_wctxs": 28, "num_trim_wctxs": 30, "num_readers": 12, "num_writers": 10, "page_bytes": 88162060, "page_cnt": 252, "page_itemcnt": 1345, "avg_item_size": 65548, "avg_page_size": 349849, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 83283584, "page_bytes_marshalled":7444088092, "page_bytes_compressed":4056649355, "compression_ratio": 1.83503, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 18528, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 200168446, "lss_data_size": 636116, "lss_used_space": 90404028, "lss_disk_size": 144498688, "lss_fragmentation": 99, "lss_num_reads": 102121, "lss_read_bs": 17324018769, "lss_blk_read_bs": 17530880000, "bytes_written": 17458585600, "bytes_incoming": 6393158632, "write_amp": 1.68, "write_amp_avg": 1.02, "lss_gc_num_reads": 102121, "lss_gc_reads_bs": 17324018769, "lss_blk_gc_reads_bs": 17530880000, "lss_gc_status": "frag 72, data: 13190637, used: 48272910, disk: 63568228, relocated: 1002513, retries: 37214, skipped: 24361, cleaned:4047033620, log:(6471830002 - 6520102912), disk:(6456534684-6520102912), activeFragRatio:30, activeMaxFragRatio:80, run:121 duration:266166 ms elapsed: 299837 ms", "lss_head_offset": 6471830002, "lss_tail_offset": 6520102912, "num_sctxs": 39, "num_free_sctxs": 31, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:16:12 [gsi-test] [] { "memory_quota": 3522410053, "count": 93818, "compacts": 54502, "purges": 0, "splits": 207, "merges": 0, "inserts": 93818, "deletes": 0, "compact_conflicts": 280, "split_conflicts": 21, "merge_conflicts": 0, "insert_conflicts": 2195, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 10, "memory_size": 75470196, "memory_size_index": 13578028, "allocated": 32439767828, "freed": 32364297632, "reclaimed": 32361871724, "reclaim_pending": 2425908, "reclaim_list_size": 2425908, "reclaim_list_count": 5, "reclaim_threshold": 1, "allocated_index": 13578028, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 944, "num_rec_allocs": 440282, "num_rec_frees": 439338, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 6149582264, "lss_data_size": 258171, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 6454275, "est_recovery_size": 38567714, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 1354803, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 1260985, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 4.53846, "currSn": 13998, "gcSn": 13971, "gcSnIntervals": "[0 7887 13972]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 24, "num_trim_wctxs": 36, "num_readers": 14, "num_writers": 9, "page_bytes": 55846896, "page_cnt": 165, "page_itemcnt": 852, "avg_item_size": 65548, "avg_page_size": 338466, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 89574080, "page_bytes_marshalled":1941116314, "page_bytes_compressed":1057550984, "compression_ratio": 1.83548, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 18192, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 1099631379, "lss_data_size": 14936734, "lss_used_space": 411348534, "lss_disk_size": 454909952, "lss_fragmentation": 96, "lss_num_reads": 22395, "lss_read_bs": 19099923109, "lss_blk_read_bs": 19531300864, "bytes_written": 19446718464, "bytes_incoming": 16559522336, "write_amp": 0.82, "write_amp_avg": 1.09, "lss_gc_num_reads": 22395, "lss_gc_reads_bs": 19099923109, "lss_blk_gc_reads_bs": 19531300864, "lss_gc_status": "frag 67, data: 98780301, used: 305421960, disk: 343390578, relocated: 17276, retries: 2645, skipped: 1142, cleaned:421389233, log:(17740476792 - 18045898752), disk:(17702508174-18045898752), activeFragRatio:30, activeMaxFragRatio:80, run:2 duration:21737 ms elapsed: 21737 ms", "lss_head_offset": 17740476792, "lss_tail_offset": 18045898752, "num_sctxs": 101, "num_free_sctxs": 91, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:16:12 [gsi-test] [] { "memory_quota": 3522410053, "count": 93236, "compacts": 54401, "purges": 0, "splits": 207, "merges": 0, "inserts": 93236, "deletes": 0, "compact_conflicts": 298, "split_conflicts": 39, "merge_conflicts": 0, "insert_conflicts": 2406, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 8, "memory_size": 79273020, "memory_size_index": 13577692, "allocated": 32621841868, "freed": 32542568848, "reclaimed": 32540601976, "reclaim_pending": 1966872, "reclaim_list_size": 1966872, "reclaim_list_count": 4, "reclaim_threshold": 1, "allocated_index": 13577692, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1002, "num_rec_allocs": 443117, "num_rec_frees": 442115, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 6111433328, "lss_data_size": 186997, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 4674925, "est_recovery_size": 35419960, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 1358759, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 1265523, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 4.81731, "currSn": 13866, "gcSn": 13839, "gcSnIntervals": "[0 7609 13840]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 26, "num_trim_wctxs": 39, "num_readers": 15, "num_writers": 10, "page_bytes": 84098084, "page_cnt": 225, "page_itemcnt": 1283, "avg_item_size": 65548, "avg_page_size": 373769, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 90098864, "page_bytes_marshalled":1757709332, "page_bytes_compressed":957689877, "compression_ratio": 1.83536, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 18912, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 1099631379, "lss_data_size": 14936734, "lss_used_space": 411348534, "lss_disk_size": 454909952, "lss_fragmentation": 96, "lss_num_reads": 22395, "lss_read_bs": 19099923109, "lss_blk_read_bs": 19531300864, "bytes_written": 19446718464, "bytes_incoming": 16560833296, "write_amp": 0.82, "write_amp_avg": 1.09, "lss_gc_num_reads": 22395, "lss_gc_reads_bs": 19099923109, "lss_blk_gc_reads_bs": 19531300864, "lss_gc_status": "frag 67, data: 98780301, used: 305421960, disk: 343390578, relocated: 17276, retries: 2645, skipped: 1142, cleaned:421389233, log:(17740476792 - 18045898752), disk:(17702508174-18045898752), activeFragRatio:30, activeMaxFragRatio:80, run:2 duration:21737 ms elapsed: 21737 ms", "lss_head_offset": 17740476792, "lss_tail_offset": 18045898752, "num_sctxs": 101, "num_free_sctxs": 91, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:16:12 [gsi-test] [9] Stop and verify... OK 2021-09-13 16:16:12 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper started 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : all deamons started 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : all deamons started 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : all deamons started 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:16:12 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper started 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : all deamons started 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : all deamons started 2021-09-13 16:16:12 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper started 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : all deamons started 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : all deamons started 2021-09-13 16:16:12 [gsi-test] [] Stop and verify... OK 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:16:12 [gsi-test] [] { "memory_quota": 3522410053, "count": 22052, "compacts": 5902, "purges": 0, "splits": 207, "merges": 0, "inserts": 22052, "deletes": 0, "compact_conflicts": 305, "split_conflicts": 30, "merge_conflicts": 0, "insert_conflicts": 4181, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 2, "memory_size": 77705212, "memory_size_index": 13577644, "allocated": 5110555756, "freed": 5032850544, "reclaimed": 5032850544, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577644, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 978, "num_rec_allocs": 71730, "num_rec_frees": 70752, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1445464496, "lss_data_size": 11477009, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 47820870, "est_recovery_size": 31220128, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 59001, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 36949, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 4.70192, "currSn": 1027, "gcSn": 1021, "gcSnIntervals": "[0 665 1022]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 4, "num_free_wctxs": 20, "num_trim_wctxs": 31, "num_readers": 4, "num_writers": 13, "page_bytes": 6358156, "page_cnt": 25, "page_itemcnt": 97, "avg_item_size": 65548, "avg_page_size": 254326, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 88535168, "page_bytes_marshalled":2322880778, "page_bytes_compressed":1265690417, "compression_ratio": 1.83527, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 24600, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 157096300, "lss_data_size": 18172997, "lss_used_space": 75746960, "lss_disk_size": 154476544, "lss_fragmentation": 76, "lss_num_reads": 74181, "lss_read_bs": 10243192755, "lss_blk_read_bs": 10556936192, "bytes_written": 10355023872, "bytes_incoming": 1445464496, "write_amp": 2.55, "write_amp_avg": 0.98, "lss_gc_num_reads": 74181, "lss_gc_reads_bs": 10243192755, "lss_blk_gc_reads_bs": 10556936192, "lss_gc_status": "frag 39, data: 27124210, used: 44805846, disk: 51885999, relocated: 8883, retries: 10, skipped: 414, cleaned:7080153, log:(1368838442 - 1413644288), disk:(1361758289-1413644288), activeFragRatio:30, activeMaxFragRatio:80, run:2 duration:467 ms elapsed: 1351 ms", "lss_head_offset": 1368838442, "lss_tail_offset": 1413644288, "num_sctxs": 33, "num_free_sctxs": 25, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:16:12 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Starting daemon Mon Sep 13 16:16:13 PDT 2021 RSS = 2867048 2021-09-13 16:16:14 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 52, data: 9843192, used: 20873076 log:(15093830 - 36098048) head:15093830 2021-09-13 16:16:14 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 72, data: 9974288, used: 36077078, disk: 37129820, relocated: 9107925, retries: 4287, skipped: 705910, cleaned:8926282148, log:(8941375978 - 8977453056), disk:(8940323236-8977453056), activeFragRatio:30, activeMaxFragRatio:80, run:292 duration:242278 ms elapsed: 305045 ms 2021-09-13 16:16:14 [gsi-test] [] { "memory_quota": 3522410053, "count": 21398, "compacts": 108, "purges": 0, "splits": 2, "merges": 0, "inserts": 21398, "deletes": 0, "compact_conflicts": 332, "split_conflicts": 4, "merge_conflicts": 0, "insert_conflicts": 4696, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 4, "memory_size": 51268976, "memory_size_index": 131192, "allocated": 3269905204, "freed": 3218636228, "reclaimed": 3218636228, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 131192, "freed_index": 0, "reclaimed_index": 0, "num_pages": 3, "items_count": 207, "total_records": 780, "num_rec_allocs": 49835, "num_rec_frees": 49055, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1402596104, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 102400, "est_recovery_size": 336294, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 56358, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 34960, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 3.76812, "currSn": 892, "gcSn": 885, "gcSnIntervals": "[0 508 886]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 13, "num_trim_wctxs": 35, "num_readers": 4, "num_writers": 9, "page_bytes": 40246472, "page_cnt": 2, "page_itemcnt": 614, "avg_item_size": 65548, "avg_page_size": 20123236, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 65433088, "page_bytes_marshalled":911845156, "page_bytes_compressed":496318778, "compression_ratio": 1.83722, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 5208, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 1099631379, "lss_data_size": 30399174, "lss_used_space": 414207809, "lss_disk_size": 483110912, "lss_fragmentation": 92, "lss_num_reads": 22544, "lss_read_bs": 19125528651, "lss_blk_read_bs": 19557916672, "bytes_written": 19474919424, "bytes_incoming": 16611436352, "write_amp": 0.82, "write_amp_avg": 1.09, "lss_gc_num_reads": 22544, "lss_gc_reads_bs": 19125528651, "lss_blk_gc_reads_bs": 19557916672, "lss_gc_status": "frag 96, data: 9266410, used: 305421960 log:(17740476792 - 18045898752) head:17702508174", "lss_head_offset": 17753628926, "lss_tail_offset": 18066595840, "num_sctxs": 101, "num_free_sctxs": 91, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:16:14 [gsi-test] [] { "memory_quota": 3522410053, "count": 20466, "compacts": 5528, "purges": 0, "splits": 207, "merges": 0, "inserts": 20466, "deletes": 0, "compact_conflicts": 311, "split_conflicts": 32, "merge_conflicts": 0, "insert_conflicts": 3957, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 1, "memory_size": 78620948, "memory_size_index": 13577804, "allocated": 4703908892, "freed": 4625287944, "reclaimed": 4616107832, "reclaim_pending": 9180112, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577804, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 992, "num_rec_allocs": 65894, "num_rec_frees": 64902, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1341505368, "lss_data_size": 8297559, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 118536557, "est_recovery_size": 28206490, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 55377, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 34911, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 4.76923, "currSn": 884, "gcSn": 880, "gcSnIntervals": "[0 504 881]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 4, "num_free_wctxs": 16, "num_trim_wctxs": 44, "num_readers": 7, "num_writers": 6, "page_bytes": 16255904, "page_cnt": 46, "page_itemcnt": 248, "avg_item_size": 65548, "avg_page_size": 353389, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 92857352, "page_bytes_marshalled":1425898886, "page_bytes_compressed":776905665, "compression_ratio": 1.83536, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 22616, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 1099631379, "lss_data_size": 28421641, "lss_used_space": 409792989, "lss_disk_size": 484032512, "lss_fragmentation": 93, "lss_num_reads": 22583, "lss_read_bs": 19130916354, "lss_blk_read_bs": 19563651072, "bytes_written": 19475841024, "bytes_incoming": 16617991152, "write_amp": 0.82, "write_amp_avg": 1.09, "lss_gc_num_reads": 22583, "lss_gc_reads_bs": 19130916354, "lss_blk_gc_reads_bs": 19563651072, "lss_gc_status": "frag 96, data: 9266410, used: 305421960 log:(17740476792 - 18045898752) head:17702508174", "lss_head_offset": 17754806790, "lss_tail_offset": 18066595840, "num_sctxs": 101, "num_free_sctxs": 92, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:16:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: starting... frag 68, data: 23700362, used: 74660555 log:(2424796382 - 2499457024) head:2297661424 2021-09-13 16:16:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: completed... frag 45, data: 16583505, used: 30222994, disk: 65703207, relocated: 1019651, retries: 37617, skipped: 24569, cleaned:4094301875, log:(6519098257 - 6550323200), disk:(6484619993-6550323200), activeFragRatio:30, activeMaxFragRatio:80, run:122 duration:269169 ms elapsed: 314747 ms 2021-09-13T16:16:20.136-07:00 [Info] memstats {"Alloc":281485880, "TotalAlloc":1351797808840, "Sys":768430328, "Lookups":0, "Mallocs":55247537,"Frees":55228728, "HeapAlloc":281485880, "HeapSys":733380608, "HeapIdle":449085440, "HeapInuse":284295168,"HeapReleased":207962112, "HeapObjects":18809,"GCSys":24365056, "LastGC":1631574979855146565,"PauseTotalNs":41559542517, "PauseNs":[2221609, 1256364, 1874782, 2822272, 88449651, 1572485, 2983040, 1982607, 2072317, 160888691, 1683607, 2031178, 1235889, 1556754, 110879415, 1860230, 1426675, 1181225, 2538074, 533232, 2310677, 92805166, 1200037, 1648613, 1061115, 1519236, 1275672, 686777, 1789931, 1536739, 1323446, 1555835, 1508186, 1290146, 4382853, 1719790, 2312083, 1659948, 1720143, 1605055, 2974504, 864241, 1810562, 2362223, 2235316, 1453955, 1371043, 3531202, 3614580, 756292, 1314177, 1751786, 3955944, 1076269, 4318984, 3174190, 2453466, 1591509, 1363419, 1082248, 1550493, 2895715, 2787569, 1361247, 2803923, 3032139, 4956717, 6926217, 3221840, 1482145, 1651521, 1609592, 2736024, 1345161, 1617460, 2464863, 1871505, 3046349, 1623050, 2508615, 2994137, 1345127, 1223472, 1409566, 2718478, 155261274, 1727814, 1507894, 1578417, 1623366, 1663167, 1648602, 1823014, 1330769, 1528282, 1479921, 919629, 1665074, 1616693, 1176869, 1600061, 920810, 1426547, 1617774, 718619, 1557782, 1267532, 1573283, 1434327, 2002266, 1241642, 1172184, 907785, 1630719, 972320, 1302687, 1188755, 1379992, 1707873, 1538730, 1148033, 1508843, 1012610, 1163306, 1501477, 995090, 1433310, 1220225, 1537606, 1328067, 1667921, 1814123, 1352399, 1216760, 748637, 1573949, 963873, 1211381, 1644407, 1830066, 1301504, 1372158, 1444010, 2367600, 1099378, 1675696, 960255, 1684823, 812179, 1619161, 917362, 1496373, 1665253, 1804938, 3048257, 1940117, 998554, 1306715, 1064816, 2354328, 1378970, 1487009, 1367492, 1572324, 1201684, 1525853, 949786, 1702029, 1011038, 1638784, 1817899, 1390817, 1024740, 2007478, 1558998, 1635131, 932424, 1920200, 1534917, 845363, 1704020, 983820, 1072233, 1594791, 1486078, 1120981, 1553571, 1338526, 1761735, 1346340, 1367248, 1071624, 1277326, 1630442, 1955627, 1748114, 1441507, 2280855, 1785282, 1296084, 1090744, 1271960, 1909467, 1358833, 4097353, 2018228, 3730359, 3224519, 7095424, 3198779, 1456932, 2657700, 989618, 1670077, 3536226, 1250885, 2648835, 1189775, 746709, 1947748, 1351988, 1486001, 2933384, 4232731, 1473117, 1646267, 1410255, 2333036, 1558736, 1927787, 1666412, 3114640, 2374659, 4237585, 2493662, 1597349, 818117, 1414047, 1947533, 3820042, 3414404, 649441, 981527, 1453430, 167869964], "NumGC":9498} Allocated: 2771947320, active: 2772049920, metadata: 19297232 (n_thp 0), resident: 2857525248, mapped: 2875088896, retained: 2592186368 2021-09-13 16:16:36 [gsi-test] [] QuotaTuner: 3522410053 -> 3170169047 2021-09-13T16:16:50.507-07:00 [Info] memstats {"Alloc":339300360, "TotalAlloc":1404991064160, "Sys":976251128, "Lookups":0, "Mallocs":58133146,"Frees":58111809, "HeapAlloc":339300360, "HeapSys":934838272, "HeapIdle":592871424, "HeapInuse":341966848,"HeapReleased":201555968, "HeapObjects":21337,"GCSys":30892032, "LastGC":1631575010433410893,"PauseTotalNs":43065126191, "PauseNs":[1821595, 1628976, 1747791, 2432495, 1013199, 1235390, 1339777, 1784356, 1067597, 1601927, 1968187, 1076933, 1784290, 3064015, 2405753, 1470704, 2937054, 44359163, 2041708, 1522120, 56577116, 3377437, 1799536, 1647733, 1730814, 1741554, 2450222, 2174420, 1584109, 1515333, 3303193, 621176, 1980786, 882279, 1512370, 1057595, 2077661, 1650257, 6412603, 203187079, 1209523, 865787, 1295484, 1386073, 2835943, 1573108, 985290, 3474727, 644463, 1517840, 1494244, 1384236, 3019208, 998223, 1715293, 1197720, 782730, 1483415, 1392739, 1874313, 2316801, 759295, 904317, 789648, 1500972, 888115, 1213396, 1142307, 2556447, 1193892, 1909646, 1827601, 1352108, 2627268, 1340139, 3116089, 726810, 982234, 2610407, 1201006, 1142282, 1247838, 1910153, 1415625, 1681790, 739041, 1771456, 1861168, 3556445, 1499489, 1519472, 1507375, 1208320, 2022465, 6040444, 1672800, 1415047, 1590198, 1118507, 1696979, 1849722, 4623261, 1890010, 123922153, 3518607, 1536685, 987845, 1741060, 1745769, 883559, 1556304, 1696436, 822712, 958178, 1982416, 3470832, 1697893, 1094035, 870060, 1125749, 1601795, 1686868, 934913, 2859036, 2806048, 929864, 829368, 3177665, 1806336, 3254753, 1336419, 1495750, 4941328, 1103233, 4245434, 1820548, 2274844, 2856858, 1837796, 1079703, 3206112, 2924046, 3813835, 969425, 1577026, 1470784, 2509497, 1672309, 3631323, 1115774, 1122764, 135500861, 89775493, 1519215, 1467319, 2691754, 1576170, 1052280, 1929500, 1466024, 2215165, 952782, 2095656, 4495487, 1400982, 3244133, 3144868, 2171580, 1095364, 3933884, 1814562, 1716730, 1606770, 1389146, 66511252, 2811408, 2889396, 67711159, 2141481, 5134751, 1793609, 154336204, 1241524, 882537, 4481891, 1357596, 1805426, 1503510, 1521546, 2005268, 164335429, 1764264, 819916, 761447, 1053583, 1446012, 3913058, 1671888, 1328136, 971145, 1333273, 1825035, 1067968, 1226585, 4843605, 2358306, 1597077, 1186184, 1904858, 1488196, 1021998, 1017293, 928987, 1204378, 1387236, 1642305, 2158258, 1068067, 823181, 741168, 1658229, 2047308, 1371051, 961700, 2597959, 1666149, 851365], "NumGC":9725} Allocated: 2773025952, active: 2773245952, metadata: 19297232 (n_thp 0), resident: 2847318016, mapped: 2864881664, retained: 2602393600 2021-09-13 16:16:55 [ss-workload] instance inse% dele% read% 2021-09-13 16:16:55 [ss-workload] 0 14% 14% 70% 2021-09-13 16:16:55 [ss-workload] 1 15% 15% 69% 2021-09-13 16:16:55 [ss-workload] 2 14% 14% 70% 2021-09-13 16:16:55 [ss-workload] 3 14% 14% 70% 2021-09-13 16:16:55 [ss-workload] 4 3% 3% 92% 2021-09-13 16:16:55 [ss-workload] 5 3% 3% 92% 2021-09-13 16:16:55 [ss-workload] 6 3% 3% 92% 2021-09-13 16:16:55 [ss-workload] 7 18% 18% 62% 2021-09-13 16:16:55 [ss-workload] 8 18% 18% 62% 2021-09-13 16:16:55 [ss-workload] 9 18% 18% 62% 2021-09-13 16:16:55 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:16:55 [ss-diag] 0 32168 31450 0 0 322 1 21106456 2 47136532 65604 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:16:55 [ss-diag] 1 31402 30434 0 0 329 1 21565292 2 63533820 65588 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:16:55 [ss-diag] 2 30594 29760 0 0 299 1 19598852 2 54744900 65588 1.000000 0.815650 1.000000 0:0:0 2021-09-13 16:16:55 [ss-diag] 3 29850 28968 0 0 323 1 21172004 2 57893924 65588 1.000000 0.815650 1.000000 0:0:0 2021-09-13 16:16:55 [ss-diag] 4 540827 539929 0 0 1422 263 93209256 208 72460028 13577836 1.000000 1.679371 1.000000 0:0:0 2021-09-13 16:16:55 [ss-diag] 5 501222 500264 0 0 1078 200 70660744 208 76388940 13578028 1.000000 0.815650 1.000000 0:0:0 2021-09-13 16:16:55 [ss-diag] 6 509073 508097 0 0 1148 219 75249104 208 77568564 13577692 1.000000 0.815650 1.000000 0:0:0 2021-09-13 16:16:55 [ss-diag] 7 78184 77240 0 0 111 27 7275828 208 75481396 13577644 1.000000 2.552276 1.000000 0:0:0 2021-09-13 16:16:55 [ss-diag] 8 52377 51319 0 0 316 1 20713168 3 69502184 131192 1.000000 0.815650 1.000000 0:0:0 2021-09-13 16:16:55 [ss-diag] 9 71227 70253 0 0 105 23 6882540 208 77445740 13577804 1.000000 0.815650 1.000000 0:0:0 Mon Sep 13 16:17:13 PDT 2021 RSS = 3081416 2021-09-13T16:17:20.734-07:00 [Info] memstats {"Alloc":489436664, "TotalAlloc":1458945416672, "Sys":976251128, "Lookups":0, "Mallocs":61054946,"Frees":61031694, "HeapAlloc":489436664, "HeapSys":934477824, "HeapIdle":442392576, "HeapInuse":492085248,"HeapReleased":358785024, "HeapObjects":23252,"GCSys":30990336, "LastGC":1631575040733934834,"PauseTotalNs":44471099444, "PauseNs":[1364022, 1936607, 1092792, 3164201, 1596139, 975122, 3141394, 1475721, 2214677, 2568547, 3537624, 2885806, 3585857, 1260771, 834598, 1376358, 1484468, 3203671, 2499661, 2200981, 1619984, 3228918, 1385194, 1777449, 2091976, 908235, 1351166, 1574313, 1696163, 1742168, 1325118, 1199932, 1557164, 3223921, 1793573, 2021103, 1230351, 954179, 1584533, 1425481, 1905711, 4261512, 1164192, 77901168, 886687, 1527416, 1552639, 1633353, 2464051, 709737, 2367310, 2143937, 893322, 1758899, 2112210, 1509238, 1269856, 1842540, 3727834, 1628476, 1799287, 3922065, 3857550, 1515003, 1590465, 1333623, 1804115, 1157044, 2807704, 2072644, 2472645, 2538125, 1102985, 977284, 1688112, 1344081, 1280320, 1442309, 1008753, 1907021, 1293291, 4486195, 2480614, 1706835, 2059688, 3119590, 1848409, 6796190, 1702297, 1806964, 2524680, 1998740, 2579892, 1932784, 690985, 9045169, 2297982, 1447054, 3203987, 156187062, 735859, 1757483, 1273344, 1255589, 2150703, 902184, 3825186, 672995, 1722646, 4750698, 1408962, 1953617, 5439457, 6040972, 994241, 4434478, 1092050, 1650936, 993387, 1286873, 1623222, 892345, 1269528, 59532800, 685208, 1083538, 2330266, 161842384, 2263073, 941422, 1559529, 1757275, 1512742, 4910567, 133215308, 4131711, 1388959, 1123555, 4090826, 1232081, 2463450, 192402996, 975715, 3123853, 1817739, 2191831, 945794, 1009116, 1562185, 1727323, 1443450, 1163246, 1838408, 698915, 5723749, 931725, 3008566, 2276918, 2985142, 1611716, 1019280, 1834521, 3994030, 3122816, 5685978, 693449, 1997637, 5251472, 3343132, 1594991, 1979855, 4224809, 2574545, 2663985, 1469297, 1609961, 2398596, 2415865, 3888835, 838205, 938618, 1211980, 6432067, 2483488, 2818788, 1494064, 3273443, 496740, 1783184, 1990666, 2747019, 2798440, 1660224, 2545441, 5519415, 878087, 1346022, 1583460, 1540248, 3737213, 1366757, 150629760, 1403879, 1066441, 1027846, 1548564, 1378009, 1710913, 2796502, 3819412, 1169519, 1236255, 1319153, 1827780, 1254178, 1565392, 3662014, 799135, 2789075, 2258055, 853714, 3565842, 926915, 641299, 1442471, 1803626, 4513282, 1636359, 1841712, 741990, 4123504, 1564629], "NumGC":9957} Allocated: 2777353288, active: 2778095616, metadata: 19297232 (n_thp 0), resident: 2888212480, mapped: 2905776128, retained: 2561499136 2021-09-13 16:17:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: starting... frag 80, data: 24177860, used: 125873766 log:(412123546 - 537997312) head:412123546 2021-09-13 16:17:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: completed... frag 83, data: 15785448, used: 94875320, disk: 99618114, relocated: 0, retries: 0, skipped: 2099884, cleaned:1220034232, log:(1632157778 - 1727426560), disk:(1627808446-1727426560), activeFragRatio:30, activeMaxFragRatio:80, run:289 duration:3429 ms elapsed: 299239 ms 2021-09-13T16:17:51.173-07:00 [Info] memstats {"Alloc":414189328, "TotalAlloc":1510910625848, "Sys":976251128, "Lookups":0, "Mallocs":63820168,"Frees":63795834, "HeapAlloc":414189328, "HeapSys":934510592, "HeapIdle":517865472, "HeapInuse":416645120,"HeapReleased":411295744, "HeapObjects":24334,"GCSys":31055872, "LastGC":1631575071122936089,"PauseTotalNs":46297936282, "PauseNs":[3056762, 787125, 1957958, 1815239, 1701275, 1575314, 1112601, 1584344, 1933196, 1095851, 2750019, 1306702, 1509926, 3441522, 4017284, 4340979, 1172580, 3470775, 1023229, 3891969, 2897201, 211972190, 1441580, 978864, 1269080, 941018, 1415763, 1729593, 1107091, 1349452, 1644594, 133098647, 2581657, 2395539, 2023072, 4535972, 1794424, 2480514, 4019366, 3385963, 2143835, 1435197, 1126170, 3521389, 3595302, 2638852, 942157, 1634020, 2891567, 1588357, 1757277, 2504251, 999344, 4086712, 2550040, 976005, 3818977, 1557187, 881429, 1839574, 1624718, 1792573, 2319506, 1623556, 4531629, 643986, 956388, 3697503, 702657, 1018981, 753066, 110675004, 1738498, 3120469, 85032270, 1360719, 1123561, 175298047, 1200753, 1241664, 1590962, 2939564, 1258814, 1297871, 2082310, 1301537, 2579485, 2490539, 1085715, 1722143, 2272449, 2563866, 2479069, 164138778, 1864752, 1835592, 3778104, 3605999, 3529895, 899678, 1971677, 1452980, 2530992, 3173706, 1342867, 738366, 1691524, 1321230, 1358308, 1722988, 1330597, 1236805, 1830509, 1314755, 1369074, 2515129, 1498761, 3239492, 4361989, 1687686, 1412601, 1327801, 1441659, 1512394, 1654906, 1322136, 3156188, 2315041, 1282508, 4317942, 1007127, 2905582, 1740571, 2111252, 1976880, 2995452, 2764747, 910690, 2560367, 1433061, 4016941, 1372040, 3754331, 4134033, 1292273, 2266588, 4669525, 1205051, 2315232, 1343032, 1819185, 1258400, 171521389, 1567336, 786303, 2825715, 1919216, 1710337, 1278305, 1558427, 1327670, 960881, 1764134, 1769031, 2161058, 1802643, 1673114, 1158293, 5255894, 1336503, 1404528, 1398071, 3069647, 1532370, 2163511, 4534980, 1541410, 2730197, 4140949, 2080959, 1523776, 490370, 1681697, 1075608, 1822212, 4112465, 959683, 1426399, 2182645, 4725174, 1583141, 1799868, 2768941, 1468194, 1285532, 2028597, 4040127, 951664, 3770715, 4348593, 1565406, 983068, 2127385, 834317, 1175846, 1556405, 962452, 1741120, 1376972, 152696529, 1423077, 1495816, 1661510, 3225412, 1768962, 973812, 2193382, 1054956, 1971135, 1740174, 186572645, 2600630, 1966042], "NumGC":10180} Allocated: 2764717976, active: 2765611008, metadata: 19297232 (n_thp 0), resident: 2854969344, mapped: 2872532992, retained: 2594742272 2021-09-13 16:18:11 [ss-workload] instance inse% dele% read% 2021-09-13 16:18:11 [ss-workload] 0 14% 14% 70% 2021-09-13 16:18:11 [ss-workload] 1 14% 14% 70% 2021-09-13 16:18:11 [ss-workload] 2 14% 14% 70% Mon Sep 13 16:18:13 PDT 2021 RSS = 3090896 2021-09-13 16:18:11 [ss-workload] 3 14% 14% 70% 2021-09-13 16:18:11 [ss-workload] 4 3% 3% 92% 2021-09-13 16:18:11 [ss-workload] 5 3% 3% 92% 2021-09-13 16:18:11 [ss-workload] 6 3% 3% 92% 2021-09-13 16:18:11 [ss-workload] 7 18% 18% 62% 2021-09-13 16:18:11 [ss-workload] 8 18% 18% 62% 2021-09-13 16:18:11 [ss-workload] 9 18% 18% 62% 2021-09-13 16:18:11 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:18:11 [ss-diag] 0 34492 33568 0 0 310 1 20319880 2 60648492 65604 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:18:11 [ss-diag] 1 33951 33089 0 0 321 1 21040908 2 56581716 65588 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:18:11 [ss-diag] 2 32859 31895 0 0 302 1 19795496 2 63272428 65588 1.000000 0.770479 1.000000 0:0:0 2021-09-13T16:18:21.545-07:00 [Info] memstats {"Alloc":357218656, "TotalAlloc":1565290289248, "Sys":976251128, "Lookups":0, "Mallocs":66757086,"Frees":66734262, "HeapAlloc":357218656, "HeapSys":934805504, "HeapIdle":574955520, "HeapInuse":359849984,"HeapReleased":376791040, "HeapObjects":22824,"GCSys":30892032, "LastGC":1631575101487503908,"PauseTotalNs":47942991232, "PauseNs":[900504, 1924586, 1309153, 877258, 2055417, 663772, 78503760, 909422, 3100684, 625221, 1602380, 4012946, 1648204, 1614856, 3391294, 116716927, 1882592, 1082459, 1372612, 1169217, 95702764, 3293421, 3977476, 1255702, 2391003, 4738451, 1514292, 3591688, 1794821, 831963, 2949487, 1267478, 4125509, 1297844, 1040036, 1992417, 1050289, 1438209, 1064354, 1635183, 2840651, 1731640, 3366399, 4469337, 3122452, 1102799, 3872607, 960507, 1481820, 917460, 2144620, 1590370, 3881027, 999248, 1187066, 4023447, 1665772, 1481490, 1076334, 1628197, 4821810, 1464560, 3212170, 2282038, 1709968, 2436140, 2408969, 446621, 312907, 1335473, 1724853, 2879107, 1923558, 6454896, 99200135, 1193926, 1594193, 1423780, 898177, 3340558, 856808, 3179290, 1512332, 1941049, 827645, 1253740, 1097111, 1369826, 3295659, 959480, 1600819, 3808430, 880804, 1133068, 1378468, 2336385, 1364862, 1787978, 1020447, 1452273, 2767274, 2253879, 1568551, 1799817, 1738774, 4037033, 2624822, 2199734, 1052208, 1719550, 1397500, 1843922, 2195088, 837490, 1292216, 725333, 7236900, 683775, 1352156, 1958963, 923702, 1837402, 1427439, 3874395, 1188783, 1303106, 1474710, 3696508, 3271201, 1617163, 1819429, 4915241, 175931175, 154153926, 1447285, 4240070, 3632258, 2216134, 1614427, 2492482, 2744560, 808330, 4940224, 1783326, 1126439, 3696511, 2840910, 4258740, 1642808, 3196532, 2139320, 1293142, 61844578, 924868, 2072835, 1899470, 2145931, 121727360, 3643018, 1413114, 756664, 72556874, 1662103, 1480933, 641821, 1630379, 1759743, 1805539, 1034209, 1475193, 1381204, 215291520, 3922251, 2014928, 2134585, 2173873, 3416589, 2783761, 1287978, 2662169, 1392985, 1836840, 1763701, 781120, 1327149, 1625126, 956982, 790386, 1661126, 2226231, 2182607, 1468427, 2017694, 3780547, 1215138, 1153767, 2412887, 1804670, 2033289, 1397959, 4447421, 2353929, 2369250, 1521049, 1500627, 1152636, 1524118, 1283559, 1883242, 1493806, 1179503, 3765390, 2924382, 5832385, 2843246, 787848, 3698392, 1149564, 704222, 735280, 2149717, 769494, 1311635, 3292037, 1492041, 1779873, 2269089, 3151652, 2322684, 4748668, 2434220], "NumGC":10411} Allocated: 2785265040, active: 2786209792, metadata: 19297232 (n_thp 0), resident: 2895327232, mapped: 2912890880, retained: 2554384384 2021-09-13 16:18:11 [ss-diag] 3 32941 32132 0 0 330 1 21630840 2 53105488 65588 1.000000 0.770479 1.000000 0:0:0 2021-09-13 16:18:11 [ss-diag] 4 624955 624125 0 0 1320 248 86523360 208 68001548 13577836 1.000000 1.675302 1.000000 0:0:0 2021-09-13 16:18:11 [ss-diag] 5 583173 582243 0 0 907 164 59452036 208 74554044 13578028 1.000000 0.770479 1.000000 0:0:0 2021-09-13 16:18:11 [ss-diag] 6 600957 600035 0 0 1012 189 66334576 208 74029660 13577692 1.000000 0.770479 1.000000 0:0:0 2021-09-13 16:18:11 [ss-diag] 7 85908 84962 0 0 82 23 5374936 208 75613564 13577644 1.000000 2.567155 1.000000 0:0:0 2021-09-13 16:18:11 [ss-diag] 8 56137 55313 0 0 645 2 42278460 3 54154720 131192 1.000000 0.770479 1.000000 0:0:0 2021-09-13 16:18:11 [ss-diag] 9 77439 76475 0 0 99 23 6489252 208 76790244 13577804 1.000000 0.770479 1.000000 0:0:0 2021-09-13T16:18:51.838-07:00 [Info] memstats {"Alloc":423170704, "TotalAlloc":1618952103368, "Sys":976251128, "Lookups":0, "Mallocs":69651121,"Frees":69625592, "HeapAlloc":423170704, "HeapSys":934477824, "HeapIdle":508739584, "HeapInuse":425738240,"HeapReleased":368500736, "HeapObjects":25529,"GCSys":30990336, "LastGC":1631575131787777223,"PauseTotalNs":49667033379, "PauseNs":[2276671, 2870021, 138925822, 1905548, 2482666, 5522449, 1539240, 2557224, 2217375, 2790378, 2601701, 1324820, 1046701, 1608926, 1571633, 1173006, 3217310, 2017594, 1759196, 3302560, 640199, 1550317, 2164324, 1746747, 1639767, 3149416, 977135, 1674905, 1677642, 1041808, 2917162, 2034755, 1407131, 2779787, 4524392, 2624577, 949720, 5586811, 1692926, 1006454, 1542079, 992862, 1189513, 1622742, 5198752, 2846867, 3463306, 1258824, 1647889, 2060524, 1245655, 1517120, 5467026, 1042491, 1369602, 2369744, 2531944, 1265296, 1290204, 1510895, 2300244, 3579572, 132455196, 1546225, 2391510, 4051585, 204739985, 1762311, 1107343, 2114996, 735965, 584177, 963857, 451254, 3278480, 902160, 1658146, 1574858, 4875496, 3365182, 2400886, 110895983, 1417124, 2745818, 4410993, 1436392, 129119413, 2402073, 1622234, 1962639, 964007, 221961244, 2302758, 2148793, 2229546, 1454898, 2117966, 149385630, 1288069, 726887, 1150977, 787892, 1011464, 4235155, 1871719, 1850250, 4016523, 3497947, 823931, 2010023, 1497617, 1282057, 563836, 1309014, 2295189, 1547125, 1750543, 1006956, 1580137, 1218548, 3363535, 3389564, 4197352, 1177172, 2689031, 1592331, 1213930, 2341978, 3435787, 3202265, 1552102, 2875015, 1601903, 1813059, 2487637, 5652890, 2328843, 2973915, 2268371, 1542552, 1906456, 3323744, 1370406, 2137533, 2141824, 3580705, 3776656, 2069701, 1844186, 1131186, 1489450, 4119174, 3300577, 1770327, 1260540, 748566, 1689888, 1565298, 4216814, 1894267, 2597815, 1604271, 1931876, 1237095, 2418472, 2838557, 1596963, 1462789, 704357, 2245477, 1205714, 4404742, 1211392, 518320, 4818152, 1677252, 1304797, 1326357, 559963, 902199, 1239237, 1176477, 2304614, 3353616, 3469965, 2035487, 1892663, 893826, 1473629, 767880, 3763803, 1595885, 1522365, 1232859, 1169734, 1335028, 2071927, 3656817, 3144519, 1487862, 1072999, 2090210, 3153659, 1559012, 1954260, 51593664, 3230514, 1788572, 984920, 1895237, 3104260, 1890277, 1098689, 2120271, 1507174, 5482896, 1676981, 1637427, 1631977, 1557160, 1074847, 129654310, 3656876, 872188, 1191460, 1620150, 3081925], "NumGC":10638} Allocated: 2874512120, active: 2875494400, metadata: 19297232 (n_thp 0), resident: 2956120064, mapped: 2973683712, retained: 2493591552 Mon Sep 13 16:19:13 PDT 2021 RSS = 2322564 2021-09-13T16:19:22.027-07:00 [Info] memstats {"Alloc":335713512, "TotalAlloc":1671311356368, "Sys":976251128, "Lookups":0, "Mallocs":72438275,"Frees":72417233, "HeapAlloc":335713512, "HeapSys":934576128, "HeapIdle":596189184, "HeapInuse":338386944,"HeapReleased":406708224, "HeapObjects":21042,"GCSys":30892032, "LastGC":1631575161966915427,"PauseTotalNs":51256478018, "PauseNs":[2175715, 1491232, 3724014, 73787904, 945462, 1855430, 1095675, 1283049, 77203312, 1540554, 1877625, 3591660, 836936, 102690640, 2332011, 1446976, 1751280, 1533103, 1405281, 84328518, 1561501, 2449731, 730121, 966484, 1743268, 1908497, 3906938, 2036719, 3221672, 3478857, 1835394, 1638039, 1244660, 1470781, 4979715, 2284094, 1768796, 4301254, 2571127, 1332497, 3649780, 2060843, 3403642, 1678916, 1610049, 2195434, 3104651, 1394602, 2609800, 1305037, 3942199, 3319232, 1639011, 1423096, 2294018, 882444, 1555609, 909038, 4100260, 2965990, 1632851, 1857634, 4735772, 1297777, 839518, 1078414, 4861639, 1211159, 2245778, 2701336, 1561954, 914165, 55548499, 1610725, 4171706, 979253, 1987063, 2704415, 2813681, 2286949, 1509998, 4473563, 1710784, 1310072, 3581035, 4359916, 1787288, 1362132, 1209609, 5542488, 2253090, 2002922, 1970012, 2861691, 2118072, 567347, 1215125, 1365700, 4344364, 1510102, 1897610, 2289681, 1577768, 933207, 1290304, 1078568, 3502914, 1555860, 2808322, 1800853, 684091, 1890393, 5611210, 1246938, 1602135, 4031165, 1901653, 1068302, 233891075, 74181860, 1464376, 2095725, 1754863, 1958030, 4935437, 2157771, 2919811, 1144734, 2529540, 1099471, 4401523, 1683738, 994636, 4524874, 4081915, 3850210, 3112893, 1588414, 1746235, 2099405, 3652925, 1454914, 1503016, 2034348, 1705988, 882577, 2334785, 1687064, 2420476, 662220, 1098735, 1612832, 1820703, 60962949, 1443248, 1433605, 1509776, 2306141, 1188714, 876521, 135149262, 1969830, 1327407, 1773753, 1674458, 236213906, 1311160, 1666565, 1269960, 1804125, 1433007, 1487036, 1123659, 1485342, 1067237, 1742188, 2329918, 577457, 807093, 2131005, 1606182, 2675733, 1778297, 2026690, 1554222, 820895, 2701865, 4403606, 974756, 726778, 860946, 1664696, 4268654, 4067533, 1477600, 1215611, 5202344, 2179702, 1399452, 3764548, 2247026, 1553743, 1127822, 2602275, 3990768, 1796626, 869493, 1759080, 4260719, 2896798, 3196598, 1383699, 1056127, 1911377, 2191365, 1382826, 2392435, 3870990, 3476324, 1723403, 1319007, 1869918, 1460636, 937705, 4408583], "NumGC":10863} Allocated: 1702701824, active: 1703714816, metadata: 19297232 (n_thp 0), resident: 1805524992, mapped: 1823088640, retained: 3644186624 2021-09-13 16:19:24 [ss-workload] instance inse% dele% read% 2021-09-13 16:19:24 [ss-workload] 0 14% 14% 70% 2021-09-13 16:19:24 [ss-workload] 1 14% 14% 70% 2021-09-13 16:19:24 [ss-workload] 2 14% 14% 70% 2021-09-13 16:19:24 [ss-workload] 3 14% 14% 70% 2021-09-13 16:19:24 [ss-workload] 4 3% 3% 92% 2021-09-13 16:19:24 [ss-workload] 5 3% 3% 92% 2021-09-13 16:19:24 [ss-workload] 6 3% 3% 92% 2021-09-13 16:19:24 [ss-workload] 7 18% 18% 62% 2021-09-13 16:19:24 [ss-workload] 8 18% 18% 62% 2021-09-13 16:19:24 [ss-workload] 9 18% 18% 62% 2021-09-13 16:19:24 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:19:24 [ss-diag] 0 37093 36197 0 0 306 1 20057688 2 58812060 65604 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:19:24 [ss-diag] 1 36586 35737 0 0 336 1 22024128 2 55728784 65588 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:19:24 [ss-diag] 2 35858 35018 0 0 296 1 19402208 2 55139292 65588 1.000000 0.775754 1.000000 0:0:0 2021-09-13 16:19:24 [ss-diag] 3 35503 34793 0 0 644 2 42212912 2 46612404 65588 1.000000 0.775754 1.000000 0:0:0 2021-09-13 16:19:24 [ss-diag] 4 707700 706834 0 0 1270 237 83245960 208 70360732 13577836 1.000000 1.700367 1.000000 0:0:0 2021-09-13 16:19:24 [ss-diag] 5 662330 661420 0 0 1327 234 86982196 208 73242188 13578028 1.000000 0.775754 1.000000 0:0:0 2021-09-13 16:19:24 [ss-diag] 6 678170 677254 0 0 964 177 63188272 208 73636228 13577692 1.000000 0.775754 1.000000 0:0:0 2021-09-13 16:19:24 [ss-diag] 7 94179 93223 0 0 210 38 13765080 208 76270084 13577644 1.000000 2.477786 1.000000 0:0:0 2021-09-13 16:19:24 [ss-diag] 8 59477 58358 0 0 341 1 22351868 3 73501132 131192 1.000000 0.775754 1.000000 0:0:0 2021-09-13 16:19:24 [ss-diag] 9 84009 83027 0 0 104 25 6816992 208 77970284 13577804 1.000000 0.775754 1.000000 0:0:0 2021-09-13 16:19:47 [gsi-test] [] QuotaTuner: 3170169047 -> 2853152142 2021-09-13T16:19:52.372-07:00 [Info] memstats {"Alloc":396200496, "TotalAlloc":1725955740680, "Sys":976251128, "Lookups":0, "Mallocs":75348299,"Frees":75325868, "HeapAlloc":396200496, "HeapSys":934412288, "HeapIdle":535511040, "HeapInuse":398901248,"HeapReleased":390381568, "HeapObjects":22431,"GCSys":30990336, "LastGC":1631575192330111895,"PauseTotalNs":52531570476, "PauseNs":[1464327, 1462128, 3011430, 1613786, 878247, 1205820, 1168574, 2358016, 1894556, 1616280, 1252262, 4105613, 77496331, 2940578, 1654509, 1646876, 1167618, 981913, 1153774, 2029748, 480795, 742501, 1520813, 3056731, 1704031, 1712675, 4441076, 1126257, 1639066, 2385515, 1361859, 1249567, 1007891, 1706936, 1692238, 1403100, 1623743, 763289, 1298563, 1794285, 1320680, 1822662, 756814, 1521979, 2501145, 1240993, 1937150, 2046382, 1081790, 1141888, 1681088, 3345743, 1462531, 1746667, 1967018, 1290645, 3166712, 3888759, 3572452, 3715712, 3927400, 1566489, 1272505, 1853619, 61491653, 1576179, 1411296, 1456786, 1223284, 1654600, 1308640, 1307481, 1258347, 1695715, 1109166, 2535006, 1255234, 176825254, 2390996, 1358307, 1843290, 873324, 3362338, 1901329, 1498870, 4096076, 87915297, 1893926, 1150537, 1248774, 3357558, 938831, 1285378, 1426774, 1522239, 1795211, 2349039, 3343459, 116608203, 5514186, 1587028, 1112317, 1683798, 1577188, 2327728, 4473108, 1772995, 1416797, 1307518, 965785, 1452519, 2574522, 4197982, 1746267, 1856477, 136564580, 1647750, 1180789, 2407015, 2613876, 1724242, 1660432, 1111631, 1292797, 1948375, 1727620, 774818, 1175981, 1245238, 1259672, 1505069, 1451080, 1326963, 2289015, 1453749, 1765187, 1450403, 4182649, 642972, 3574335, 1604928, 1314556, 3079775, 1497277, 7333125, 1462373, 1091175, 2143131, 4243601, 800468, 3912043, 3886213, 3499354, 3695479, 2258406, 1268430, 3743062, 1372037, 2686255, 1005795, 3659915, 2204890, 1510721, 3937810, 1214790, 1624550, 2200912, 929949, 3526415, 2349812, 945457, 1035430, 1336983, 2346721, 2329572, 1897282, 3363462, 1225940, 983046, 130889496, 1104495, 1131546, 1211377, 1893434, 3544582, 2152135, 1111950, 4373637, 2312182, 4059938, 2150344, 1065820, 1313816, 570222, 3663249, 3218714, 1306316, 1577595, 2199603, 1421482, 930661, 1461054, 3374161, 1311881, 1550992, 5115469, 915722, 1482132, 1106188, 3018050, 2387720, 1071459, 1608906, 1664539, 2206391, 3421018, 1790631, 1149619, 4027105, 3008146, 828201, 2569943, 1046237, 5329017, 1608256, 1161961, 1525976, 1535753, 3528049, 3197441, 1436074, 2694038, 2414496, 23654233, 11714876], "NumGC":11098} Allocated: 2120534976, active: 2125811712, metadata: 19297232 (n_thp 0), resident: 2241810432, mapped: 2259374080, retained: 3207901184 Mon Sep 13 16:20:13 PDT 2021 RSS = 2717436 2021-09-13 16:20:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 229, "num_active_sctxs": 48, "num_free_sctxs": 188, "num_trim_sctxs": 93, "num_wctxs": 172, "num_free_wctxs": 150, "num_active_wctxs": 22, "num_trim_wctxs": 545, "num_readers": 76, "num_writers": 67, "cpu": 9.56265, "total_core": 16, "threads": 47, "assigned_quota": 2853152142, "current_quota": 2853152142, "memory_resident": 0, "memory_jemalloc": 2397437952, "memory_jemalloc_allocated": 2327355160, "memory_in_use": 2396913664, "memory_stats_size_total": 2491208517, "memory_stats_size_global": 747024436, "memory_stats_size_page": 678648968, "memory_stats_size_index": 68348152, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 169496, "reclaim_pending": 18687396, "buf_memused": 1725524001, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 18, "num_active_smr": 12, "num_pending_smr": 5, "num_pages": 1052, "items_count": 2077, "total_records": 9308, "total_ops": 520454, "compacts": 331195, "purges": 0, "lss_data_size": 188654118, "lss_used_space": 689236973, "lss_fragmentation": 72, "lss_num_reads": 172252, "lss_read_bs": 43347072772, "lss_blk_read_bs": 44198408192, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 44002729984, "bytes_incoming": 24290122360, "write_amp_avg": 0.86, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 120 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9308, "resident_ratio": 100.00000, "cleaner_num_reads": 172252, "cleaner_read_bs": 43347072772, "cleaner_blk_read_bs": 44198408192, "recovery_num_lssctx": 5, "recovery_num_sctxs": 28, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 8, "recovery_num_trim_sctxs": 8, "recovery_buf_memused": 3145728, "recovery_data_size": 48142564, "recovery_used_space": 196721964, "recovery_lss_blk_read_bs": 18882883584, "recovery_lss_read_bs": 18193908345, "recovery_lss__num_reads": 142099, "recovery_bytes_written": 18421755904, "recovery_lss_frag": 75, "recovery_cleaner_num_reads": 142099, "recovery_cleaner_read_bs": 18193908345, "recovery_cleaner_blk_read_bs": 18882883584, "mvcc_purge_ratio": 4.48146 }2021-09-13T16:20:22.757-07:00 [Info] memstats {"Alloc":446449688, "TotalAlloc":1768423290760, "Sys":976251128, "Lookups":0, "Mallocs":77645583,"Frees":77624599, "HeapAlloc":446449688, "HeapSys":934838272, "HeapIdle":485621760, "HeapInuse":449216512,"HeapReleased":307011584, "HeapObjects":20984,"GCSys":30990336, "LastGC":1631575222754708106,"PauseTotalNs":54721589545, "PauseNs":[16967777, 19209153, 24477133, 15805801, 27751532, 25150528, 15425300, 21927700, 37639932, 30123859, 28892856, 24973901, 31241401, 36873620, 106948668, 20729186, 15853926, 85111231, 19127743, 25395543, 69286282, 28172446, 25263126, 18674852, 25712236, 30416642, 25061793, 37970172, 22517544, 28212199, 44028963, 30555096, 43183102, 43202159, 28707155, 7538362, 2031299, 1583458, 1080427, 1836002, 1119647, 1274320, 2149733, 1848311, 1384745, 1409624, 823030, 1192726, 1135862, 3053379, 2627538, 1276734, 2480436, 1398255, 1139010, 1656375, 1807666, 1216047, 1953283, 555025, 562412, 4344294, 1976898, 2072918, 1978039, 1350458, 3304043, 1209744, 680588, 1195365, 2084641, 2580310, 2054682, 1424827, 600553, 1774335, 1975287, 864226, 1441422, 1980645, 928227, 2948201, 3421792, 2934133, 939033, 4232761, 1627853, 3530974, 1237945, 1043206, 4390655, 2142442, 1088696, 1550230, 150421788, 2371782, 2385600, 1668088, 2318907, 3400660, 1910853, 2018391, 4664711, 1647613, 2057666, 1921686, 3325200, 5520548, 1861824, 3040758, 4193058, 4761430, 2265042, 2150436, 1531124, 2301631, 4247985, 1658238, 3796830, 1027984, 1115919, 1030625, 4163617, 2524952, 1609877, 2340333, 1510861, 1765282, 613420, 2672059, 1019743, 2378231, 2685439, 1918343, 1930608, 3385500, 1435096, 1108413, 21292837, 5146620, 2127523, 1678933, 1681853, 2656591, 1588275, 2146164, 1393867, 2379355, 81405186, 2406095, 1015237, 992231, 673287, 958929, 249170649, 1632979, 1077776, 1381902, 5224306, 2223805, 56728107, 940471, 1572969, 833381, 1332527, 1122213, 2905078, 1314312, 2577662, 1319058, 2012041, 106090097, 2140020, 1362598, 1173434, 904434, 1313578, 871036, 115939816], "NumGC":11277} Allocated: 2304302872, active: 2305261568, metadata: 21423056 (n_thp 0), resident: 2394865664, mapped: 2410332160, retained: 3308601344 2021-09-13 16:20:30 [gsi-test] [9] CreateRecoveryPoint() - DO!!! 2021-09-13 16:20:33 [gsi-test] [2] CreateRecoveryPoint() - DO!!! 2021-09-13 16:20:34 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 16:20:34 [ss-workload] instance inse% dele% read% 2021-09-13 16:20:34 [ss-workload] 0 14% 14% 70% 2021-09-13 16:20:35 [gsi-test] [6] CreateRecoveryPoint() - DO!!! 2021-09-13 16:20:34 [ss-workload] 1 14% 14% 70% 2021-09-13 16:20:34 [ss-workload] 2 14% 14% 70% 2021-09-13 16:20:36 [gsi-test] [8] CreateRecoveryPoint() - DO!!! 2021-09-13 16:20:34 [ss-workload] 3 14% 14% 70% 2021-09-13 16:20:34 [ss-workload] 4 3% 3% 92% 2021-09-13 16:20:34 [ss-workload] 5 3% 3% 92% 2021-09-13 16:20:34 [ss-workload] 6 3% 3% 92% 2021-09-13 16:20:39 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 16:20:34 [ss-workload] 7 18% 18% 62% 2021-09-13 16:20:40 [gsi-test] [5] CreateRecoveryPoint() - DO!!! 2021-09-13 16:20:40 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 16:20:34 [ss-workload] 8 18% 18% 62% 2021-09-13 16:20:34 [ss-workload] 9 18% 18% 62% 2021-09-13 16:20:34 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:20:41 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13 16:20:41 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: starting... frag 70, data: 111128535, used: 381706809 log:(17319087559 - 17700794368) head:17272556560 2021-09-13 16:20:41 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: completed... frag 56, data: 166045485, used: 385636775, disk: 492963287, relocated: 188688, retries: 20971, skipped: 8919, cleaned:5136537716, log:(22455625275 - 22841262080), disk:(22348298793-22841262080), activeFragRatio:30, activeMaxFragRatio:80, run:15 duration:289650 ms elapsed: 301297 ms 2021-09-13 16:20:34 [ss-diag] 0 39754 38808 0 0 325 1 21303100 2 62090660 65604 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:20:34 [ss-diag] 1 38747 37835 0 0 340 1 22286320 2 59860444 65588 1.000000 0.000000 1.000000 0:0:0 2021-09-13 16:20:34 [ss-diag] 2 38604 37700 0 0 314 1 20582072 2 59335900 65588 1.000000 0.974342 1.000000 0:0:0 2021-09-13 16:20:45 [gsi-test] [3] CreateRecoveryPoint() - DO!!! 2021-09-13 16:20:34 [ss-diag] 3 37703 36875 0 0 328 1 21499744 2 54351532 65588 1.000000 0.974342 1.000000 0:0:0 2021-09-13 16:20:34 [ss-diag] 4 784046 783106 0 0 1242 235 81410616 208 75214180 13577836 1.000000 1.671733 1.000000 0:0:0 2021-09-13 16:20:34 [ss-diag] 5 739192 738198 0 0 913 170 59845324 208 78750812 13578028 1.000000 0.974342 1.000000 0:0:0 2021-09-13 16:20:34 [ss-diag] 6 757916 756838 0 0 1542 280 101075016 208 84254092 13577692 1.000000 0.974342 1.000000 0:0:0 2021-09-13 16:20:34 [ss-diag] 7 101615 100685 0 0 134 31 8783432 208 74566332 13577644 1.000000 2.369286 1.000000 0:0:0 2021-09-13 16:20:34 [ss-diag] 8 62687 61427 0 0 600 2 39328800 4 82815284 196780 1.000000 0.974342 1.000000 0:0:0 2021-09-13 16:20:34 [ss-diag] 9 91924 90896 0 0 233 44 15272684 208 80986852 13577804 1.000000 0.974342 1.000000 0:0:0 2021-09-13 16:20:48 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: starting... frag 83, data: 17168228, used: 102952661 log:(31125803 - 134078464) head:19777471 2021-09-13 16:20:48 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: completed... frag 49, data: 33190778, used: 65657605, disk: 108402362, relocated: 116, retries: 100, skipped: 23, cleaned:542368720, log:(573494523 - 639152128), disk:(530749766-639152128), activeFragRatio:30, activeMaxFragRatio:80, run:12 duration:24070 ms elapsed: 566848 ms 2021-09-13T16:20:53.027-07:00 [Info] memstats {"Alloc":322341208, "TotalAlloc":1808210644344, "Sys":976251128, "Lookups":0, "Mallocs":80002035,"Frees":79972844, "HeapAlloc":322341208, "HeapSys":934445056, "HeapIdle":609771520, "HeapInuse":324673536,"HeapReleased":168124416, "HeapObjects":29191,"GCSys":30994432, "LastGC":1631575252938615651,"PauseTotalNs":57418750460, "PauseNs":[1011898, 1867341, 1240390, 1343422, 2186469, 1709659, 620570, 1019033, 1422664, 1312548, 2301631, 1846643, 1640111, 1892888, 1156825, 4563769, 1697238, 1788886, 678524, 4337677, 1597377, 3794211, 2790931, 1219544, 2170094, 1483681, 2351101, 4709413, 839687, 2618121, 1512678, 956559, 1223290, 3438103, 1760055, 1227909, 3914273, 3554893, 711145, 3947500, 1683952, 1383149, 1086989, 1831118, 798297, 940863, 2274916, 1791621, 2441412, 1223625, 3024465, 1954812, 3911159, 3727907, 1303083, 1617351, 2521412, 1498215, 898442, 1196442, 1431065, 1462783, 1943072, 1813358, 1483607, 2370077, 1958218, 744660, 2076300, 1345934, 1463622, 1784866, 4464126, 1546377, 1436610, 1077623, 2678405, 2764663, 1149080, 2401249, 1275408, 1171565, 2590480, 1232811, 995662, 4008286, 1359563, 75928422, 31737910, 4337013, 5481039, 7427832, 9027641, 1784067, 1327092, 6107873, 122227902, 3696051, 15934446, 11936707, 13626585, 28318658, 9813186, 25317725, 11884524, 28180539, 46338840, 18708665, 75856483, 11777352, 81995850, 36532996, 21750586, 162710710, 20370534, 41104220, 25504145, 50718313, 36178303, 11190028, 19769600, 3231661, 28832219, 32949824, 56185175, 17372123, 50477128, 35821252, 52226238, 50653211, 42298984, 50993302, 26906609, 11603396, 36795179, 38368797, 52546520, 16195284, 21903312, 8569513, 17190221, 5651893, 905095, 105781551, 899716, 131429432, 34097143, 28374195, 17400853, 21928535, 38970938, 36487206, 23039376, 32535405, 42195336, 27270681, 15290490, 38493586, 36603081, 49744762, 19106690, 11922005, 16684339, 8424202, 20424814, 39180355], "NumGC":11443} Allocated: 2813830264, active: 2814775296, metadata: 21423056 (n_thp 0), resident: 3002519552, mapped: 3017986048, retained: 2700947456 2021-09-13 16:21:00 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: starting... frag 35, data: 31554481, used: 48988046 log:(1361758289 - 1411534848) head:1360687093 2021-09-13 16:21:00 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: completed... frag 30, data: 32619907, used: 46898115, disk: 54189861, relocated: 1838479, retries: 1750, skipped: 56770, cleaned:1106344459, log:(2468102748 - 2515251200), disk:(2461061339-2515251200), activeFragRatio:30, activeMaxFragRatio:80, run:270 duration:69892 ms elapsed: 299578 ms 2021-09-13 16:21:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: starting... frag 61, data: 19989722, used: 52493884 log:(56644036 - 109137920) head:26914903 2021-09-13 16:21:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: completed... frag 69, data: 33542016, used: 110149545, disk: 122888361, relocated: 106, retries: 91, skipped: 20, cleaned:336518515, log:(393162551 - 503312384), disk:(380424023-503312384), activeFragRatio:30, activeMaxFragRatio:80, run:10 duration:20998 ms elapsed: 592871 ms 2021-09-13 16:21:02 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 80, data: 8400740, used: 42659451 log:(10894438890 - 10937495552) head:10893386148 2021-09-13 16:21:02 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 88, data: 5843648, used: 49376772, disk: 50298390, relocated: 23082832, retries: 79625, skipped: 6004292, cleaned:10568704000, log:(21463142890 - 21513441280), disk:(21463142890-21513441280), activeFragRatio:30, activeMaxFragRatio:80, run:237 duration:287784 ms elapsed: 300332 ms Mon Sep 13 16:21:13 PDT 2021 RSS = 3503700 2021-09-13 16:21:14 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 69, data: 11351192, used: 36998678 log:(8941375978 - 8978374656) head:8940323236 2021-09-13 16:21:14 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 42, data: 17448812, used: 30154262, disk: 32653172, relocated: 28525083, retries: 12081, skipped: 2099194, cleaned:9248464686, log:(18189840664 - 18220388352), disk:(18187735180-18220388352), activeFragRatio:30, activeMaxFragRatio:80, run:295 duration:255279 ms elapsed: 299960 ms 2021-09-13 16:21:15 [gsi-test] [] Stop and verify... 2021-09-13 16:21:16 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper stopped 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: starting... frag 45, data: 16583505, used: 30222994 log:(6519098257 - 6550323200) head:6484619993 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: completed... frag 77, data: 11329425, used: 50195574, disk: 64945809, relocated: 3129658, retries: 57996, skipped: 26125, cleaned:3914583434, log:(10433681691 - 10484699136), disk:(10419753327-10484699136), activeFragRatio:30, activeMaxFragRatio:80, run:122 duration:266806 ms elapsed: 300005 ms 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:21:16 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : All daemons stopped 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper stopped 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:21:16 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : All daemons stopped 2021-09-13 16:21:16 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper stopped 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:21:16 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : All daemons stopped 2021-09-13 16:21:16 [gsi-test] [0] Stop and verify... 2021-09-13 16:21:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:21:18 [gsi-test] [0] Stop and verify... OK 2021-09-13 16:21:18 [gsi-test] [1] Stop and verify... 2021-09-13 16:21:19 [gsi-test] [] { "memory_quota": 2853152142, "count": 16664, "compacts": 85, "purges": 0, "splits": 3, "merges": 0, "inserts": 16664, "deletes": 0, "compact_conflicts": 198, "split_conflicts": 10, "merge_conflicts": 0, "insert_conflicts": 2821, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 69955684, "memory_size_index": 196812, "allocated": 2742419996, "freed": 2672464312, "reclaimed": 2672464312, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 196812, "freed_index": 0, "reclaimed_index": 0, "num_pages": 4, "items_count": 208, "total_records": 1064, "num_rec_allocs": 41799, "num_rec_frees": 40735, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1092291872, "lss_data_size": 33115831, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 33115831, "est_recovery_size": 2750918, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 49825, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 33161, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 5.11538, "currSn": 1178, "gcSn": 1175, "gcSnIntervals": "[0 581 1144 1176]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 24, "num_trim_wctxs": 43, "num_readers": 8, "num_writers": 8, "page_bytes": 41491884, "page_cnt": 2, "page_itemcnt": 633, "avg_item_size": 65548, "avg_page_size": 20745942, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 54023832, "page_bytes_marshalled":669510930, "page_bytes_compressed":364412915, "compression_ratio": 1.83723, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 9072, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 444289543, "lss_data_size": 33517519, "lss_used_space": 29800029, "lss_disk_size": 80187392, "lss_fragmentation": 0, "lss_num_reads": 107, "lss_read_bs": 877084312, "lss_blk_read_bs": 878346240, "bytes_written": 885493760, "bytes_incoming": 1092291872, "write_amp": 7.60, "write_amp_avg": 0.81, "lss_gc_num_reads": 107, "lss_gc_reads_bs": 877084312, "lss_blk_gc_reads_bs": 878346240, "lss_gc_status": "frag 0, data: 33115831, used: 24016477, disk: 89907734, relocated: 70, retries: 55, skipped: 11, cleaned:282199208, log:(855693731 - 879710208), disk:(789802474-879710208), activeFragRatio:30, activeMaxFragRatio:80, run:3 duration:9240 ms elapsed: 17460 ms", "lss_head_offset": 855693731, "lss_tail_offset": 879710208, "num_sctxs": 34, "num_free_sctxs": 25, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:21:21 [gsi-test] [1] Stop and verify... OK 2021-09-13 16:21:21 [gsi-test] [2] Stop and verify... 2021-09-13 16:21:22 [gsi-test] [2] Stop and verify... OK 2021-09-13 16:21:22 [gsi-test] [3] Stop and verify... 2021-09-13T16:21:23.327-07:00 [Info] memstats {"Alloc":476099592, "TotalAlloc":1842451816048, "Sys":976251128, "Lookups":0, "Mallocs":82656187,"Frees":82626242, "HeapAlloc":476099592, "HeapSys":935264256, "HeapIdle":456916992, "HeapInuse":478347264,"HeapReleased":301473792, "HeapObjects":29945,"GCSys":30961664, "LastGC":1631575283306577691,"PauseTotalNs":60927248788, "PauseNs":[34073081, 36133015, 16686123, 159019114, 36320787, 48273571, 48392703, 34126666, 35149413, 19481459, 28517060, 41442517, 80198825, 8960593, 46519253, 940964, 32817350, 232590894, 2254481, 2702902, 1614827, 3786277, 1760170, 64153136, 2831570, 1477724, 112306395, 44087396, 14130739, 890360, 3448658, 1801295, 931639, 33724585, 3484732, 184187995, 42598020, 234409986, 79658409, 102834630, 21491868, 25064152, 79769792, 26617737, 21550428, 18983153, 5923754, 30146256, 28593434, 58289165, 16366576, 34472245, 19629814, 229468853, 21041368, 24835678, 142820550, 23061665, 24192618, 25805157, 21179858, 41338539, 19750156, 19775223, 19477197, 33360438, 17923914, 3900090, 28680027, 22432940, 21419722, 36807388, 17987342, 8960399, 18818591, 29560800, 19353440, 12972497, 29801286, 5148625, 825266, 115799720, 1362347, 4008480, 2518865, 1430544, 120803862, 532989, 2785204, 4861434, 1564951, 1087779, 477258, 1542339, 1132908, 1248735, 1131255, 2441986, 1210108, 3335619, 1160133, 787170, 1246669, 1321202, 1150514, 1713107, 1927976, 1447793, 1521832, 1333715, 1234126, 1256489, 947523, 1435673, 543379, 1089413, 1952675, 1441972, 1684929, 1646480, 1190907, 427330, 1027022, 1498504, 1234722, 1577573, 1124905, 1613120, 1066995, 1375031, 1727456, 1709279, 1286912, 1626189, 1686690, 1860292, 1950966, 722386, 1355581], "NumGC":11582} Allocated: 3162638568, active: 3163660288, metadata: 21423056 (n_thp 0), resident: 3255050240, mapped: 3270598656, retained: 2448334848 2021-09-13 16:21:24 [gsi-test] [] { "memory_quota": 2853152142, "count": 16409, "compacts": 83, "purges": 0, "splits": 2, "merges": 0, "inserts": 16409, "deletes": 0, "compact_conflicts": 144, "split_conflicts": 6, "merge_conflicts": 0, "insert_conflicts": 2468, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 3, "memory_size": 61562356, "memory_size_index": 131176, "allocated": 2640930636, "freed": 2579368280, "reclaimed": 2579368280, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 131176, "freed_index": 0, "reclaimed_index": 0, "num_pages": 3, "items_count": 208, "total_records": 937, "num_rec_allocs": 40253, "num_rec_frees": 39316, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1075577132, "lss_data_size": 30616307, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 41373387, "est_recovery_size": 2013236, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 48947, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 32538, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 4.50481, "currSn": 1186, "gcSn": 1182, "gcSnIntervals": "[0 579 1155 1183]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 20, "num_trim_wctxs": 53, "num_readers": 7, "num_writers": 6, "page_bytes": 39918732, "page_cnt": 2, "page_itemcnt": 609, "avg_item_size": 65548, "avg_page_size": 19959366, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 59135600, "page_bytes_marshalled":619957904, "page_bytes_compressed":337433474, "compression_ratio": 1.83727, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 6472, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 305720775, "lss_data_size": 30886863, "lss_used_space": 41849088, "lss_disk_size": 94269440, "lss_fragmentation": 26, "lss_num_reads": 86, "lss_read_bs": 704590745, "lss_blk_read_bs": 704761856, "bytes_written": 698249216, "bytes_incoming": 1075577132, "write_amp": 4.99, "write_amp_avg": 0.65, "lss_gc_num_reads": 86, "lss_gc_reads_bs": 704590745, "lss_blk_gc_reads_bs": 704761856, "lss_gc_status": "frag 18, data: 30616307, used: 37781760, disk: 103635469, relocated: 82, retries: 72, skipped: 22, cleaned:263237577, log:(656400128 - 694181888), disk:(590546419-694181888), activeFragRatio:30, activeMaxFragRatio:80, run:4 duration:8640 ms elapsed: 14645 ms", "lss_head_offset": 656400128, "lss_tail_offset": 694181888, "num_sctxs": 30, "num_free_sctxs": 20, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:21:24 [gsi-test] [] { "memory_quota": 2853152142, "count": 16001, "compacts": 81, "purges": 0, "splits": 2, "merges": 0, "inserts": 16001, "deletes": 0, "compact_conflicts": 174, "split_conflicts": 3, "merge_conflicts": 0, "insert_conflicts": 2469, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 65889724, "memory_size_index": 131176, "allocated": 2640001612, "freed": 2574111888, "reclaimed": 2574111888, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 131176, "freed_index": 0, "reclaimed_index": 0, "num_pages": 3, "items_count": 208, "total_records": 1003, "num_rec_allocs": 40240, "num_rec_frees": 39237, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1048833548, "lss_data_size": 31975163, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 228394021, "est_recovery_size": 401900, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 47843, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 31842, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 4.82212, "currSn": 1083, "gcSn": 1079, "gcSnIntervals": "[0 466 1047 1080]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 18, "num_trim_wctxs": 54, "num_readers": 7, "num_writers": 4, "page_bytes": 23728376, "page_cnt": 1, "page_itemcnt": 362, "avg_item_size": 65548, "avg_page_size": 23728376, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 60839448, "page_bytes_marshalled":1562545046, "page_bytes_compressed":850475368, "compression_ratio": 1.83726, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 7360, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 903794736, "lss_data_size": 95363402, "lss_used_space": 691900539, "lss_disk_size": 764407808, "lss_fragmentation": 86, "lss_num_reads": 42733, "lss_read_bs": 25531746985, "lss_blk_read_bs": 26186096640, "bytes_written": 26131558400, "bytes_incoming": 22802772692, "write_amp": 1.12, "write_amp_avg": 1.03, "lss_gc_num_reads": 42733, "lss_gc_reads_bs": 25531746985, "lss_blk_gc_reads_bs": 26186096640, "lss_gc_status": "frag 77, data: 122627574, used: 545677845, disk: 577079499, relocated: 35983, retries: 3006, skipped: 1598, cleaned:415750027, log:(22871375302 - 23417839616), disk:(22840760117-23417839616), activeFragRatio:30, activeMaxFragRatio:80, run:2 duration:35284 ms elapsed: 35284 ms", "lss_head_offset": 22871375302, "lss_tail_offset": 23417839616, "num_sctxs": 74, "num_free_sctxs": 59, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:21:24 [gsi-test] [3] Stop and verify... OK 2021-09-13 16:21:24 [gsi-test] [4] Stop and verify... 2021-09-13 16:21:24 [gsi-test] [4] Stop and verify... OK 2021-09-13 16:21:24 [gsi-test] [5] Stop and verify... 2021-09-13 16:21:24 [gsi-test] [5] Stop and verify... OK 2021-09-13 16:21:24 [gsi-test] [6] Stop and verify... 2021-09-13 16:21:25 [gsi-test] [6] Stop and verify... OK 2021-09-13 16:21:25 [gsi-test] [7] Stop and verify... 2021-09-13 16:21:26 [gsi-test] [] { "memory_quota": 2853152142, "count": 15868, "compacts": 80, "purges": 0, "splits": 2, "merges": 0, "inserts": 15868, "deletes": 0, "compact_conflicts": 199, "split_conflicts": 2, "merge_conflicts": 0, "insert_conflicts": 2768, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 4, "memory_size": 68186424, "memory_size_index": 131176, "allocated": 2583553560, "freed": 2515367136, "reclaimed": 2515367136, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 131176, "freed_index": 0, "reclaimed_index": 0, "num_pages": 3, "items_count": 208, "total_records": 1038, "num_rec_allocs": 39376, "num_rec_frees": 38338, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1040115664, "lss_data_size": 29194195, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 224570730, "est_recovery_size": 467506, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 47490, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 31622, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 4.99038, "currSn": 1089, "gcSn": 1086, "gcSnIntervals": "[0 468 1050 1087]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 23, "num_trim_wctxs": 54, "num_readers": 9, "num_writers": 6, "page_bytes": 39918732, "page_cnt": 2, "page_itemcnt": 609, "avg_item_size": 65548, "avg_page_size": 19959366, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 61102920, "page_bytes_marshalled":1537112202, "page_bytes_compressed":836638995, "compression_ratio": 1.83725, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 10160, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 903794736, "lss_data_size": 83381548, "lss_used_space": 691900539, "lss_disk_size": 764407808, "lss_fragmentation": 87, "lss_num_reads": 42733, "lss_read_bs": 25531746985, "lss_blk_read_bs": 26186096640, "bytes_written": 26131558400, "bytes_incoming": 22846689852, "write_amp": 1.12, "write_amp_avg": 1.02, "lss_gc_num_reads": 42733, "lss_gc_reads_bs": 25531746985, "lss_blk_gc_reads_bs": 26186096640, "lss_gc_status": "frag 77, data: 122627574, used: 545677845, disk: 577079499, relocated: 35983, retries: 3006, skipped: 1598, cleaned:415750027, log:(22871375302 - 23417839616), disk:(22840760117-23417839616), activeFragRatio:30, activeMaxFragRatio:80, run:2 duration:35284 ms elapsed: 35284 ms", "lss_head_offset": 22871375302, "lss_tail_offset": 23417839616, "num_sctxs": 74, "num_free_sctxs": 62, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:21:26 [gsi-test] [] { "memory_quota": 2853152142, "count": 133246, "compacts": 132970, "purges": 0, "splits": 207, "merges": 0, "inserts": 133246, "deletes": 0, "compact_conflicts": 246, "split_conflicts": 28, "merge_conflicts": 0, "insert_conflicts": 2416, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 4, "memory_size": 92378844, "memory_size_index": 13577836, "allocated": 67458414836, "freed": 67366035992, "reclaimed": 67364855328, "reclaim_pending": 1180664, "reclaim_list_size": 787280, "reclaim_list_count": 2, "reclaim_threshold": 0, "allocated_index": 13577836, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1202, "num_rec_allocs": 896350, "num_rec_frees": 895148, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 8734008808, "lss_data_size": 1150624, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 57531200, "est_recovery_size": 41580346, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 2059686, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 1926440, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 5.77885, "currSn": 20943, "gcSn": 20917, "gcSnIntervals": "[0 9651 20176 20918]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 35, "num_trim_wctxs": 56, "num_readers": 13, "num_writers": 14, "page_bytes": 165049864, "page_cnt": 449, "page_itemcnt": 2518, "avg_item_size": 65548, "avg_page_size": 367594, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 105303936, "page_bytes_marshalled":14663214384, "page_bytes_compressed":7990888666, "compression_ratio": 1.83499, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12096, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 177215614, "lss_data_size": 1683408, "lss_used_space": 92064816, "lss_disk_size": 157859840, "lss_fragmentation": 98, "lss_num_reads": 204947, "lss_read_bs": 32256869929, "lss_blk_read_bs": 32683077632, "bytes_written": 32437223424, "bytes_incoming": 8734008808, "write_amp": 2.28, "write_amp_avg": 1.20, "lss_gc_num_reads": 204947, "lss_gc_reads_bs": 32256869929, "lss_blk_gc_reads_bs": 32683077632, "lss_gc_status": "frag 77, data: 11329425, used: 50195574, disk: 64945809, relocated: 3129658, retries: 57996, skipped: 26125, cleaned:3914583434, log:(10433681691 - 10484699136), disk:(10419753327-10484699136), activeFragRatio:30, activeMaxFragRatio:80, run:122 duration:266806 ms elapsed: 300005 ms", "lss_head_offset": 10433681691, "lss_tail_offset": 10484699136, "num_sctxs": 48, "num_free_sctxs": 38, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:21:26 [gsi-test] [] { "memory_quota": 2853152142, "count": 130662, "compacts": 123895, "purges": 0, "splits": 207, "merges": 0, "inserts": 130662, "deletes": 0, "compact_conflicts": 280, "split_conflicts": 21, "merge_conflicts": 0, "insert_conflicts": 2196, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 10, "memory_size": 94213964, "memory_size_index": 13578028, "allocated": 64796519948, "freed": 64702305984, "reclaimed": 64699683456, "reclaim_pending": 2622528, "reclaim_list_size": 1573536, "reclaim_list_count": 4, "reclaim_threshold": 1, "allocated_index": 13578028, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 207, "total_records": 1230, "num_rec_allocs": 864717, "num_rec_frees": 863487, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 8564632776, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 63015, "est_recovery_size": 50699430, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 2063069, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 1932407, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 5.94203, "currSn": 19933, "gcSn": 19907, "gcSnIntervals": "[0 7887 19143 19908]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 37, "num_trim_wctxs": 54, "num_readers": 15, "num_writers": 14, "page_bytes": 154168896, "page_cnt": 413, "page_itemcnt": 2352, "avg_item_size": 65548, "avg_page_size": 373290, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 104777280, "page_bytes_marshalled":3460952570, "page_bytes_compressed":1885789602, "compression_ratio": 1.83528, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 11648, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 903794736, "lss_data_size": 83381548, "lss_used_space": 691900539, "lss_disk_size": 764407808, "lss_fragmentation": 87, "lss_num_reads": 42733, "lss_read_bs": 25531746985, "lss_blk_read_bs": 26186096640, "bytes_written": 26131558400, "bytes_incoming": 22846755400, "write_amp": 1.12, "write_amp_avg": 1.02, "lss_gc_num_reads": 42733, "lss_gc_reads_bs": 25531746985, "lss_blk_gc_reads_bs": 26186096640, "lss_gc_status": "frag 77, data: 122627574, used: 545677845, disk: 577079499, relocated: 35983, retries: 3006, skipped: 1598, cleaned:415750027, log:(22871375302 - 23417839616), disk:(22840760117-23417839616), activeFragRatio:30, activeMaxFragRatio:80, run:2 duration:35284 ms elapsed: 35284 ms", "lss_head_offset": 22871375302, "lss_tail_offset": 23417839616, "num_sctxs": 74, "num_free_sctxs": 63, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:21:26 [gsi-test] [7] Stop and verify... OK 2021-09-13 16:21:26 [gsi-test] [8] Stop and verify... 2021-09-13 16:21:26 [gsi-test] [] { "memory_quota": 2853152142, "count": 130806, "compacts": 125589, "purges": 0, "splits": 207, "merges": 0, "inserts": 130806, "deletes": 0, "compact_conflicts": 298, "split_conflicts": 39, "merge_conflicts": 0, "insert_conflicts": 2406, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 9, "memory_size": 92902908, "memory_size_index": 13577692, "allocated": 65821748116, "freed": 65728845208, "reclaimed": 65727796216, "reclaim_pending": 1048992, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577692, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1210, "num_rec_allocs": 878643, "num_rec_frees": 877433, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 8574071688, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 63015, "est_recovery_size": 55945590, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 2074207, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 1943401, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 5.81731, "currSn": 19756, "gcSn": 19734, "gcSnIntervals": "[0 7609 18901 19735]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 36, "num_trim_wctxs": 64, "num_readers": 15, "num_writers": 13, "page_bytes": 154693280, "page_cnt": 419, "page_itemcnt": 2360, "avg_item_size": 65548, "avg_page_size": 369196, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 105563936, "page_bytes_marshalled":3240837610, "page_bytes_compressed":1765906256, "compression_ratio": 1.83523, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 11728, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 903794736, "lss_data_size": 83381548, "lss_used_space": 691900539, "lss_disk_size": 764407808, "lss_fragmentation": 87, "lss_num_reads": 42733, "lss_read_bs": 25531746985, "lss_blk_read_bs": 26186096640, "bytes_written": 26131558400, "bytes_incoming": 22848197456, "write_amp": 1.12, "write_amp_avg": 1.02, "lss_gc_num_reads": 42733, "lss_gc_reads_bs": 25531746985, "lss_blk_gc_reads_bs": 26186096640, "lss_gc_status": "frag 77, data: 122627574, used: 545677845, disk: 577079499, relocated: 35983, retries: 3006, skipped: 1598, cleaned:415750027, log:(22871375302 - 23417839616), disk:(22840760117-23417839616), activeFragRatio:30, activeMaxFragRatio:80, run:2 duration:35284 ms elapsed: 35284 ms", "lss_head_offset": 22871375302, "lss_tail_offset": 23417839616, "num_sctxs": 74, "num_free_sctxs": 65, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:21:26 [gsi-test] [] { "memory_quota": 2853152142, "count": 29337, "compacts": 12642, "purges": 0, "splits": 207, "merges": 0, "inserts": 29337, "deletes": 0, "compact_conflicts": 305, "split_conflicts": 30, "merge_conflicts": 0, "insert_conflicts": 4181, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 3, "memory_size": 91140000, "memory_size_index": 13577644, "allocated": 8098420596, "freed": 8007280596, "reclaimed": 8006428240, "reclaim_pending": 852356, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577644, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 207, "total_records": 1183, "num_rec_allocs": 110593, "num_rec_frees": 109410, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1922981676, "lss_data_size": 15227601, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 52508968, "est_recovery_size": 29121856, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 79499, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 50162, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 5.71498, "currSn": 1363, "gcSn": 1360, "gcSnIntervals": "[0 665 1308 1361]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 18, "num_trim_wctxs": 50, "num_readers": 3, "num_writers": 8, "page_bytes": 25629268, "page_cnt": 67, "page_itemcnt": 391, "avg_item_size": 65548, "avg_page_size": 382526, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 101643328, "page_bytes_marshalled":4443350404, "page_bytes_compressed":2421224418, "compression_ratio": 1.83517, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 19072, "num_total_smr": 2, "num_active_smr": 3, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 64389151, "lss_data_size": 23366041, "lss_used_space": 82474112, "lss_disk_size": 183361536, "lss_fragmentation": 71, "lss_num_reads": 151849, "lss_read_bs": 20696559928, "lss_blk_read_bs": 21337387008, "bytes_written": 20852891648, "bytes_incoming": 1922981676, "write_amp": 2.62, "write_amp_avg": 1.34, "lss_gc_num_reads": 151849, "lss_gc_reads_bs": 20696559928, "lss_blk_gc_reads_bs": 21337387008, "lss_gc_status": "frag 44, data: 29583734, used: 53507340, disk: 57187068, relocated: 119611, retries: 91, skipped: 3655, cleaned:52649949, log:(2520752697 - 2574438400), disk:(2517251332-2574438400), activeFragRatio:30, activeMaxFragRatio:80, run:13 duration:4316 ms elapsed: 14936 ms", "lss_head_offset": 2520752697, "lss_tail_offset": 2574438400, "num_sctxs": 32, "num_free_sctxs": 21, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:21:28 [gsi-test] [8] Stop and verify... OK 2021-09-13 16:21:28 [gsi-test] [9] Stop and verify... 2021-09-13 16:21:28 [gsi-test] [9] Stop and verify... OK 2021-09-13 16:21:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper started 2021-09-13 16:21:28 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : all deamons started 2021-09-13 16:21:28 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : all deamons started 2021-09-13 16:21:28 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : all deamons started 2021-09-13 16:21:28 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:21:28 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:21:28 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:21:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper started 2021-09-13 16:21:28 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : all deamons started 2021-09-13 16:21:28 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : all deamons started 2021-09-13 16:21:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper started 2021-09-13 16:21:28 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : all deamons started 2021-09-13 16:21:28 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : all deamons started 2021-09-13 16:21:28 [gsi-test] [] Stop and verify... OK 2021-09-13 16:21:28 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:21:29 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:21:29 [gsi-test] [] { "memory_quota": 2853152142, "count": 28112, "compacts": 142, "purges": 0, "splits": 3, "merges": 0, "inserts": 28112, "deletes": 0, "compact_conflicts": 386, "split_conflicts": 14, "merge_conflicts": 0, "insert_conflicts": 5210, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 5, "memory_size": 85296748, "memory_size_index": 196780, "allocated": 4267035056, "freed": 4181738308, "reclaimed": 4181738308, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 196780, "freed_index": 0, "reclaimed_index": 0, "num_pages": 4, "items_count": 208, "total_records": 1298, "num_rec_allocs": 65014, "num_rec_frees": 63716, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1842685376, "lss_data_size": 9787558, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 97875580, "est_recovery_size": 401842, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 74089, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 45977, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 6.24038, "currSn": 1228, "gcSn": 1223, "gcSnIntervals": "[0 508 1170 1224]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 20, "num_trim_wctxs": 47, "num_readers": 4, "num_writers": 7, "page_bytes": 12323024, "page_cnt": 1, "page_itemcnt": 188, "avg_item_size": 65548, "avg_page_size": 12323024, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 89305440, "page_bytes_marshalled":1829329266, "page_bytes_compressed":995722323, "compression_ratio": 1.83719, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 10448, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 913189926, "lss_data_size": 67989144, "lss_used_space": 691900539, "lss_disk_size": 764407808, "lss_fragmentation": 90, "lss_num_reads": 42733, "lss_read_bs": 25531746985, "lss_blk_read_bs": 26186096640, "bytes_written": 26131558400, "bytes_incoming": 22899652636, "write_amp": 1.12, "write_amp_avg": 1.02, "lss_gc_num_reads": 42733, "lss_gc_reads_bs": 25531746985, "lss_blk_gc_reads_bs": 26186096640, "lss_gc_status": "frag 77, data: 122627574, used: 545677845, disk: 577079499, relocated: 35983, retries: 3006, skipped: 1598, cleaned:415750027, log:(22871375302 - 23417839616), disk:(22840760117-23417839616), activeFragRatio:30, activeMaxFragRatio:80, run:2 duration:35284 ms elapsed: 35284 ms", "lss_head_offset": 22871375302, "lss_tail_offset": 23417839616, "num_sctxs": 74, "num_free_sctxs": 58, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:21:30 [gsi-test] [] { "memory_quota": 2853152142, "count": 27136, "compacts": 11805, "purges": 0, "splits": 207, "merges": 0, "inserts": 27136, "deletes": 0, "compact_conflicts": 311, "split_conflicts": 32, "merge_conflicts": 0, "insert_conflicts": 3957, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 1, "memory_size": 94610852, "memory_size_index": 13577804, "allocated": 7576356236, "freed": 7481745384, "reclaimed": 7481351936, "reclaim_pending": 393448, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577804, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1236, "num_rec_allocs": 103444, "num_rec_frees": 102208, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1778710528, "lss_data_size": 6721829, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 67218290, "est_recovery_size": 32932152, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 74255, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 47119, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 5.94231, "currSn": 1244, "gcSn": 1242, "gcSnIntervals": "[0 504 1176 1243]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 20, "num_trim_wctxs": 70, "num_readers": 3, "num_writers": 10, "page_bytes": 34150508, "page_cnt": 84, "page_itemcnt": 521, "avg_item_size": 65548, "avg_page_size": 406553, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 105963592, "page_bytes_marshalled":2707658776, "page_bytes_compressed":1475379210, "compression_ratio": 1.83523, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 15432, "num_total_smr": 2, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 921940584, "lss_data_size": 67608021, "lss_used_space": 706900091, "lss_disk_size": 779407360, "lss_fragmentation": 90, "lss_num_reads": 42735, "lss_read_bs": 25532056848, "lss_blk_read_bs": 26188193792, "bytes_written": 26146557952, "bytes_incoming": 22910271412, "write_amp": 1.12, "write_amp_avg": 1.02, "lss_gc_num_reads": 42735, "lss_gc_reads_bs": 25532056848, "lss_blk_gc_reads_bs": 26188193792, "lss_gc_status": "frag 88, data: 63349916, used: 560677397 log:(22871375302 - 23432839168) head:22858551571", "lss_head_offset": 22871375302, "lss_tail_offset": 23432839168, "num_sctxs": 74, "num_free_sctxs": 60, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:21:48 [ss-workload] instance inse% dele% read% 2021-09-13 16:21:48 [ss-workload] 0 14% 14% 70% 2021-09-13 16:21:48 [ss-workload] 1 14% 14% 70% 2021-09-13T16:21:53.562-07:00 [Info] memstats {"Alloc":473260400, "TotalAlloc":1880941976336, "Sys":976251128, "Lookups":0, "Mallocs":85769653,"Frees":85741529, "HeapAlloc":473260400, "HeapSys":934313984, "HeapIdle":458563584, "HeapInuse":475750400,"HeapReleased":399491072, "HeapObjects":28124,"GCSys":31059968, "LastGC":1631575313561969322,"PauseTotalNs":63234479067, "PauseNs":[2550339, 1782831, 1241470, 1246308, 478752, 673676, 1467849, 1067179, 1245550, 1292568, 1046827, 957953, 2421281, 508260, 1536541, 1021460, 1828346, 1552089, 1120230, 1596615, 1166201, 1198153, 1385862, 2481324, 936814, 1170252, 1434468, 2653377, 1315008, 1860939, 1833124, 872375, 1135657, 1597007, 1120124, 1917933, 1895994, 1413639, 1423151, 1574259, 1190258, 1368351, 11694668, 14541041, 40606965, 16106928, 8457809, 15556846, 23631262, 23161816, 22794875, 23032933, 11617434, 19315108, 30151473, 26682157, 25110422, 23206268, 18583913, 41172534, 15253403, 30252168, 23585140, 2753532, 8891951, 2569840, 1971913, 16427122, 6726272, 9637256, 10624723, 19275652, 13157345, 10765815, 23659768, 11668756, 18218940, 20220969, 18970581, 36277641, 27849175, 26999587, 18687768, 17311305, 39798538, 15358002, 21343319, 16604277, 2229905, 1755254, 3278363, 1464510, 3404357, 784298, 2420840, 1455751, 4761952, 5428672, 3562789, 8048539, 1048779, 12991985, 23344795, 5463572, 10203935, 2547937, 9651287, 16445868, 19472378, 174005210, 16742788, 2086442, 13704319, 1512441, 173522540, 135707299, 5399780, 905395, 2380763, 1064887, 1382405, 10575376, 6057758, 18824937, 10623675, 6890789, 119288008, 18760810, 2531333, 3046521, 13043198, 2939006, 184045281, 880201, 2438676, 32756957, 21007883, 1725557, 1469368, 22168847, 4530930, 1307704, 3398070, 1304556, 648795, 1346856, 3340252, 2253742, 112584930, 26489279, 807917, 1078842, 1795674, 1119385, 1504376, 56599151], "NumGC":11738} Allocated: 3535929952, active: 3537285120, metadata: 21423056 (n_thp 0), resident: 3662884864, mapped: 3678351360, retained: 2040582144 2021-09-13 16:21:48 [ss-workload] 2 14% 14% 70% 2021-09-13 16:21:48 [ss-workload] 3 14% 14% 70% 2021-09-13 16:21:48 [ss-workload] 4 3% 3% 92% 2021-09-13 16:21:48 [ss-workload] 5 3% 3% 92% 2021-09-13 16:21:48 [ss-workload] 6 3% 3% 92% 2021-09-13 16:21:48 [ss-workload] 7 18% 18% 62% 2021-09-13 16:21:48 [ss-workload] 8 18% 18% 62% 2021-09-13 16:21:48 [ss-workload] 9 18% 18% 62% 2021-09-13 16:21:48 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:21:48 [ss-diag] 0 42279 40735 0 0 633 2 41491884 4 101437924 196812 1.000000 7.603115 1.000000 0:0:0 2021-09-13 16:21:48 [ss-diag] 1 41453 40363 0 0 201 1 13175148 4 71660332 196764 1.000000 4.985635 1.000000 0:0:0 2021-09-13 16:21:48 [ss-diag] 2 41576 40244 0 0 665 2 43589420 4 87531252 196780 1.000000 1.123850 1.000000 0:0:0 2021-09-13 16:21:48 [ss-diag] 3 40633 39409 0 0 203 1 13306244 4 80448548 196764 1.000000 1.123850 1.000000 0:0:0 2021-09-13 16:21:48 [ss-diag] 4 996830 995610 0 0 3146 555 206214008 208 93560388 13577836 1.000000 2.278527 1.000000 0:0:0 2021-09-13 16:21:48 [ss-diag] 5 971357 970171 0 0 3712 624 243314176 208 91329884 13578028 1.000000 1.123850 1.000000 0:0:0 2021-09-13 16:21:48 [ss-diag] 6 981361 980155 0 0 3716 648 243576368 208 92640620 13577692 1.000000 1.123850 1.000000 0:0:0 2021-09-13 16:21:48 [ss-diag] 7 118283 117049 0 0 454 74 29758792 208 94488668 13577644 1.000000 2.619372 1.000000 0:0:0 2021-09-13 16:21:48 [ss-diag] 8 66193 64639 0 0 262 1 17173576 4 102086124 196780 1.000000 1.123850 1.000000 0:0:0 2021-09-13 16:21:48 [ss-diag] 9 111913 110674 0 0 325 55 21303100 208 94807936 13577804 1.000000 1.123850 1.000000 0:0:0 Mon Sep 13 16:22:13 PDT 2021 RSS = 3699656 2021-09-13T16:22:23.773-07:00 [Info] memstats {"Alloc":537676032, "TotalAlloc":1925772619032, "Sys":976251128, "Lookups":0, "Mallocs":89502555,"Frees":89464817, "HeapAlloc":537676032, "HeapSys":934772736, "HeapIdle":394633216, "HeapInuse":540139520,"HeapReleased":372391936, "HeapObjects":37738,"GCSys":30896128, "LastGC":1631575343640212363,"PauseTotalNs":64366517095, "PauseNs":[801739, 96697411, 1217989, 1443395, 3391977, 1622106, 1809702, 1516276, 5218691, 1760023, 2934351, 1638231, 1221714, 1505107, 2186399, 1066853, 1383961, 105152109, 1291047, 961929, 2017738, 1796229, 1138127, 1817571, 2364490, 970328, 2286593, 1356922, 1521151, 1822859, 1744854, 62255543, 1873611, 5298463, 3670774, 1623029, 1719440, 6990652, 1611638, 1855450, 1292780, 1587206, 4218236, 4036371, 110452035, 1286400, 3470126, 1293333, 4659451, 1811080, 3852707, 1412638, 921309, 2835681, 385687, 1501625, 1364478, 2848257, 1966961, 983925, 1973736, 2854216, 2609084, 1352537, 1705336, 1116559, 1776262, 4622122, 1930881, 1438100, 1207108, 1571876, 1055426, 1674671, 1069043, 5223177, 4712785, 3394989, 1243545, 3560185, 2969627, 803096, 1767741, 2659531, 1550457, 4678474, 798419, 2972504, 1432300, 2908363, 4062901, 1589652, 1430088, 1212696, 3334940, 1315414, 582824, 1567811, 1544064, 3539713, 4892585, 5413872, 118021787, 1524160, 1229396, 33889799, 1785476, 4509224, 2180198, 791142, 4483411, 4981693, 1423685, 542132, 3231713, 1432925, 1725621, 5461907, 825596, 1093718, 2137665, 1652665, 3560488, 2138036, 1893401, 3649032, 3346848, 1154033, 1481642, 1217787, 1531589, 5288177, 2449277, 1297207, 1096542, 5644049, 4576243, 604682, 2017315, 1525883, 1569214, 730154, 1329465, 155851058, 1332064, 413874, 2765498, 4042613, 4116547, 1319265, 2478770, 796071, 1255290, 1616673, 1411783, 1601415, 729734, 1797030, 1257684, 2309441, 1337361, 1747106, 4521289, 1989157, 3007352, 3737684, 705304, 1018570, 53982356, 20719067, 1805106, 957230, 2850800, 930513, 1127239, 1904467, 1457117, 1558463, 816922, 1202158, 1693859, 3638126, 1355527], "NumGC":11921} Allocated: 3596156296, active: 3597594624, metadata: 21433808 (n_thp 0), resident: 3738030080, mapped: 3753947136, retained: 1964986368 2021-09-13 16:22:37 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: starting... frag 82, data: 18276956, used: 102248120 log:(1632157778 - 1734799360) head:1632157778 2021-09-13 16:22:37 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: completed... frag 87, data: 16834504, used: 138051010, disk: 148053934, relocated: 0, retries: 0, skipped: 4716865, cleaned:1243304100, log:(2875461878 - 3014037504), disk:(2865983570-3014037504), activeFragRatio:30, activeMaxFragRatio:80, run:288 duration:3238 ms elapsed: 299368 ms 2021-09-13T16:22:54.102-07:00 [Info] memstats {"Alloc":383227920, "TotalAlloc":1968583257408, "Sys":976251128, "Lookups":0, "Mallocs":93068881,"Frees":93040868, "HeapAlloc":383227920, "HeapSys":934445056, "HeapIdle":548773888, "HeapInuse":385671168,"HeapReleased":403038208, "HeapObjects":28013,"GCSys":30994432, "LastGC":1631575374088520238,"PauseTotalNs":65745387320, "PauseNs":[2399680, 1845906, 106620298, 2973897, 4826734, 86258052, 1223547, 812796, 833515, 1596198, 1758868, 919507, 370089, 4982113, 2090318, 922221, 1798444, 1170395, 1521381, 755994, 878245, 4394292, 3351131, 1740256, 1476826, 2994298, 1245940, 1374821, 1899524, 108537971, 940486, 14515390, 3316805, 3766186, 43216744, 878477, 1321686, 83780516, 11976842, 8406177, 3587931, 1421831, 903786, 1187004, 1174445, 1268494, 4334781, 1287619, 2538135, 1299907, 3951504, 1941639, 3782394, 1412263, 1906684, 1090833, 1262448, 1504012, 1669555, 1276346, 2765034, 3295843, 3878153, 387983, 1583915, 926703, 3583347, 1305800, 3943989, 1072117, 3587246, 2097407, 974204, 5624720, 1900953, 3681953, 1832738, 1772627, 912594, 2026453, 1266945, 2126437, 1870948, 1712441, 1380006, 6658078, 1038354, 2260320, 2009998, 5377990, 1814813, 3749850, 821721, 2102100, 1098309, 948311, 3056302, 4550883, 1771662, 1278649, 2465537, 882542, 1193549, 5352797, 5288704, 1892975, 4013667, 2723374, 2240497, 2742249, 62487976, 5559630, 1867331, 1584208, 874377, 1471634, 4870165, 799858, 903065, 1503094, 4466220, 1830061, 1467990, 6584071, 3907275, 1704729, 744595, 3465760, 1494561, 1464634, 2286997, 1056769, 3338275, 5344504, 128311377, 3074334, 1379253, 3597233, 3760143, 3952632, 1387888, 62783139, 1794615, 952156, 4120241, 1974291, 889690, 951285, 1484726, 396613, 781059, 448964, 1397321, 1735405, 1243234, 1446502, 1734482, 1644910, 851283, 765772, 1547170, 287078916, 2694695, 1887090, 2344493, 6036187, 3871206, 7645361, 1534326, 2259135, 1077281, 2350767, 3430622, 2201121, 1627810, 1211735, 2726502, 1483804, 4587713], "NumGC":12100} Allocated: 3627321904, active: 3628941312, metadata: 21449808 (n_thp 0), resident: 3774918656, mapped: 3790360576, retained: 1928572928 2021-09-13 16:22:56 [gsi-test] [] QuotaTuner: 2853152142 -> 2567836927 2021-09-13 16:22:59 [ss-workload] instance inse% dele% read% 2021-09-13 16:22:59 [ss-workload] 0 14% 14% 70% 2021-09-13 16:22:59 [ss-workload] 1 14% 14% 70% 2021-09-13 16:22:59 [ss-workload] 2 14% 14% 70% 2021-09-13 16:22:59 [ss-workload] 3 14% 14% 70% 2021-09-13 16:22:59 [ss-workload] 4 3% 3% 92% 2021-09-13 16:22:59 [ss-workload] 5 3% 3% 92% 2021-09-13 16:22:59 [ss-workload] 6 3% 3% 92% 2021-09-13 16:22:59 [ss-workload] 7 18% 18% 62% 2021-09-13 16:22:59 [ss-workload] 8 18% 18% 62% 2021-09-13 16:22:59 [ss-workload] 9 18% 18% 62% 2021-09-13 16:22:59 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:22:59 [ss-diag] 0 45083 43773 0 0 275 1 18025700 4 86082812 196812 1.000000 0.192312 1.000000 0:0:0 2021-09-13 16:22:59 [ss-diag] 1 43395 42035 0 0 267 1 17501316 4 89362756 196764 1.000000 2.203127 1.000000 0:0:0 2021-09-13 16:22:59 [ss-diag] 2 43887 42421 0 0 238 1 15600424 4 96315692 196780 1.000000 1.016091 1.000000 0:0:0 2021-09-13 16:22:59 [ss-diag] 3 42638 41090 0 0 253 1 16583644 4 101694484 196764 1.000000 1.016091 1.000000 0:0:0 2021-09-13 16:22:59 [ss-diag] 4 1276653 1275459 0 0 5219 861 342095012 208 91855868 13577836 1.000000 2.480872 1.000000 0:0:0 2021-09-13 16:22:59 [ss-diag] 5 1259264 1258034 0 0 4078 698 267304744 208 94214348 13578028 1.000000 1.016091 1.000000 0:0:0 2021-09-13 16:22:59 [ss-diag] 6 1282026 1280842 0 0 5173 862 339079804 208 91199092 13577692 1.000000 1.016091 1.000000 0:0:0 2021-09-13 16:22:59 [ss-diag] 7 141292 139986 0 0 443 69 29037764 208 99209340 13577644 1.000000 3.776450 1.000000 0:0:0 2021-09-13 16:22:59 [ss-diag] 8 70587 68907 0 0 262 1 17173576 4 110347844 196780 1.000000 1.016091 1.000000 0:0:0 2021-09-13 16:22:59 [ss-diag] 9 135936 134649 0 0 279 43 18287892 208 97957376 13577804 1.000000 1.016091 1.000000 0:0:0 Mon Sep 13 16:23:13 PDT 2021 RSS = 3738744 2021-09-13T16:23:24.523-07:00 [Info] memstats {"Alloc":320560800, "TotalAlloc":2013069339864, "Sys":976251128, "Lookups":0, "Mallocs":96758127,"Frees":96735464, "HeapAlloc":320560800, "HeapSys":934445056, "HeapIdle":610762752, "HeapInuse":323682304,"HeapReleased":393814016, "HeapObjects":22663,"GCSys":30896128, "LastGC":1631575404381938124,"PauseTotalNs":67121298486, "PauseNs":[3233146, 7619445, 1162550, 3909676, 3648673, 1376425, 37117067, 1252257, 624429, 4729294, 1401086, 1952296, 948033, 110801471, 5482847, 4875192, 1718592, 4771372, 573400, 4147160, 1047048, 1922551, 2874898, 5844698, 1717286, 2890084, 3018368, 942471, 1787022, 1369082, 1226019, 1342659, 1575885, 1844552, 1490892, 1247086, 569076, 959872, 1022378, 201876355, 2828730, 1156380, 1102940, 1316453, 1619713, 10105828, 1478594, 976183, 2238491, 2398826, 659172, 1927771, 586230, 4510796, 2851366, 1003064, 2081103, 1692782, 4175140, 1337837, 3349935, 1655850, 887427, 3253203, 1550673, 2726168, 1003007, 3184013, 2063229, 24679720, 1461722, 3845175, 1505846, 1287134, 115768362, 8723359, 626102, 1409278, 6860763, 1857811, 1738783, 1603745, 674417, 64852412, 1516377, 1235706, 919123, 1102118, 1313701, 1225574, 1356012, 864189, 4056868, 1586788, 1321811, 1344792, 1713471, 1308604, 3327245, 3238883, 6258744, 3167501, 3001859, 2952903, 1929888, 1121518, 41671352, 42578478, 2119128, 2212671, 890360, 1151197, 1601731, 1847703, 2117465, 960783, 3157582, 899614, 3654645, 2907312, 6173456, 1078203, 1494841, 5294146, 1523873, 1512181, 49052399, 91881677, 1211186, 3158062, 970989, 1981060, 1683709, 3241248, 566769, 1584331, 730947, 1462836, 1574690, 3174138, 738150, 768495, 1969511, 1716888, 2171456, 3523635, 1461319, 1657445, 2038373, 1860051, 1538015, 1721369, 934507, 1188610, 974962, 450576, 1441154, 1756943, 480128, 2928617, 150295684, 642889, 1050797, 1786112, 3131063, 48932815, 1569371, 3602758, 4602728, 25968753, 1929197, 2465402, 784882, 3091750, 904674, 2912250, 1384169, 3198320, 848875, 1711932, 2716117, 2052961, 1938908, 1839558, 1716334], "NumGC":12285} Allocated: 3637867240, active: 3639472128, metadata: 21601104 (n_thp 0), resident: 3763384320, mapped: 3778818048, retained: 1940115456 2021-09-13T16:23:54.779-07:00 [Info] memstats {"Alloc":435317648, "TotalAlloc":2056668498792, "Sys":976251128, "Lookups":0, "Mallocs":100337104,"Frees":100303154, "HeapAlloc":435317648, "HeapSys":934150144, "HeapIdle":496623616, "HeapInuse":437526528,"HeapReleased":334921728, "HeapObjects":33950,"GCSys":31027200, "LastGC":1631575434732103874,"PauseTotalNs":69092702936, "PauseNs":[111551589, 1014673, 2769976, 1628248, 3436478, 1254616, 937025, 2592285, 939541, 3266248, 951161, 3421437, 1177740, 1905375, 1084677, 1417805, 3065411, 1474800, 1551956, 2008667, 1037550, 2715235, 1419328, 6342581, 1072989, 65358967, 1078256, 1666570, 3035748, 72202828, 1776179, 1054311, 1374639, 1500538, 4435372, 98216165, 43530380, 1261231, 2285943, 674862, 975638, 1748937, 1919144, 4532582, 3753032, 2783528, 3588314, 3520203, 1003261, 1744521, 4164104, 833759, 1877290, 92385379, 1233352, 607613, 139999094, 2769730, 621655, 1404567, 1189495, 2919180, 2214213, 977758, 1321744, 1299466, 3837278, 1609863, 2112768, 1542402, 595988, 662531, 3710808, 2426301, 2450213, 1448158, 1228847, 1442725, 1222354, 4399500, 4135270, 1240262, 954484, 1384215, 2315771, 1356816, 2617022, 653095, 960000, 1746284, 1580514, 1619869, 2015773, 3645202, 1524787, 1203021, 1291709, 1175164, 1757141, 2710897, 2143234, 5873478, 1336023, 4508468, 1075772, 1846347, 1576374, 2611210, 773926, 1858201, 1641928, 1632940, 872377, 1262598, 2453266, 96703068, 2805900, 1306887, 2063387, 3016251, 1759099, 3881779, 43460542, 265436007, 1048646, 1079285, 1616532, 1825243, 1577480, 4630602, 2369143, 1803606, 2214729, 1448194, 1216985, 3187338, 100241979, 4801340, 47329929, 129637467, 1195302, 1191262, 924797, 1177977, 32277870, 4576900, 1360226, 3666385, 4445112, 2407431, 1171257, 5772306, 811129, 1685792, 880894, 868441, 3509955, 1695328, 823613, 1511662, 1419930, 37073848, 1607421, 1253394, 1381107, 3747393, 730266, 1287302, 169983453, 1868245, 1260273, 94954148, 3681517, 3411350, 5046023, 784300, 750910, 1014446, 409205, 4404749], "NumGC":12465} Allocated: 3623320792, active: 3624837120, metadata: 21634640 (n_thp 0), resident: 3785977856, mapped: 3801489408, retained: 2051661824 2021-09-13 16:24:13 [ss-workload] instance inse% dele% read% Mon Sep 13 16:24:13 PDT 2021 RSS = 3731828 2021-09-13 16:24:13 [ss-workload] 0 14% 14% 70% 2021-09-13 16:24:13 [ss-workload] 1 14% 14% 70% 2021-09-13 16:24:13 [ss-workload] 2 14% 14% 70% 2021-09-13 16:24:13 [ss-workload] 3 14% 14% 70% 2021-09-13 16:24:13 [ss-workload] 4 3% 3% 92% 2021-09-13 16:24:13 [ss-workload] 5 3% 3% 92% 2021-09-13 16:24:13 [ss-workload] 6 3% 3% 92% 2021-09-13 16:24:13 [ss-workload] 7 18% 18% 62% 2021-09-13 16:24:13 [ss-workload] 8 18% 18% 62% 2021-09-13 16:24:13 [ss-workload] 9 18% 18% 62% 2021-09-13 16:24:13 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:24:13 [ss-diag] 0 47135 45643 0 0 277 1 18156796 4 98019988 196812 1.000000 1.722465 1.000000 0:0:0 2021-09-13 16:24:13 [ss-diag] 1 45809 44373 0 0 278 1 18222344 4 94346644 196764 1.000000 1.167203 1.000000 0:0:0 2021-09-13 16:24:13 [ss-diag] 2 46238 44766 0 0 239 1 15665972 4 96707908 196780 1.000000 0.930642 1.000000 0:0:0 2021-09-13 16:24:13 [ss-diag] 3 45006 43390 0 0 260 1 17042480 4 106152996 196764 1.000000 0.930642 1.000000 0:0:0 2021-09-13 16:24:13 [ss-diag] 4 1564605 1563417 0 0 4279 730 280479892 208 91462916 13577836 1.000000 2.274966 1.000000 0:0:0 2021-09-13 16:24:13 [ss-diag] 5 1540866 1539654 0 0 4526 757 296670248 208 93034020 13578028 1.000000 0.930642 1.000000 0:0:0 2021-09-13 16:24:13 [ss-diag] 6 1585260 1584040 0 0 5059 839 331607332 208 93559172 13577692 1.000000 0.930642 1.000000 0:0:0 2021-09-13 16:24:13 [ss-diag] 7 162076 160738 0 0 306 49 20057688 208 101308028 13577644 1.000000 3.866695 1.000000 0:0:0 2021-09-13T16:24:25.153-07:00 [Info] memstats {"Alloc":486101768, "TotalAlloc":2097661656144, "Sys":976251128, "Lookups":0, "Mallocs":103762731,"Frees":103726533, "HeapAlloc":486101768, "HeapSys":934150144, "HeapIdle":445874176, "HeapInuse":488275968,"HeapReleased":410140672, "HeapObjects":36198,"GCSys":31059968, "LastGC":1631575465053787943,"PauseTotalNs":71449545512, "PauseNs":[1719688, 1419123, 5031235, 53889535, 3485101, 1825853, 3236757, 5014024, 2480399, 160838133, 1694185, 4405934, 1425335, 83669707, 8475570, 5371958, 735481, 1366031, 1858326, 759024, 113163300, 1575347, 17865502, 3745909, 1221589, 93720469, 875711, 3813354, 63098956, 3293022, 3402797, 73672794, 1313445, 159682814, 802503, 1561765, 69856278, 796812, 137844532, 92853151, 8114049, 74350209, 3704416, 1010458, 4152184, 902865, 1338140, 3338395, 89118780, 4247295, 1838313, 126655493, 2711784, 1293492, 3368850, 1400986, 50891238, 1611627, 1816527, 1415794, 205975890, 1950210, 442246, 2645390, 1702366, 3874703, 5034632, 1945390, 1042128, 3657080, 1596746, 1646429, 1461448, 1124775, 4869276, 1153701, 3576793, 2149425, 1354477, 1487480, 1777715, 2506791, 3818133, 1303372, 2283710, 5472757, 6009711, 4550742, 1524509, 5547148, 4021108, 961948, 2738627, 1500987, 4344773, 2846185, 855162, 3539877, 2595762, 1571782, 1095048, 1390546, 1654267, 986405, 2907966, 1056240, 3729029, 3534043, 2475366, 811347, 1278621, 6266566, 1651500, 1480084, 994347, 494847, 96758495, 66504058, 892936, 2548943, 1339704, 1333859, 1326360, 2070104, 1277969, 3013446, 1549716, 74856385, 1661261, 2338900, 968903, 1607276, 1429251, 1753384, 36704018, 23798181, 1588022, 900491, 948712, 598645, 2503160, 5237169, 1129003, 1072968, 1733244, 3597820, 5043965, 3967165, 42097388, 2966866, 1838702, 1174841, 1404136, 1427821, 2134929, 1536536, 848010, 2405528, 5726207, 2586095, 2734414, 1359356, 887074, 3897292, 1073326, 1790514, 2531383, 1677710, 4720360, 1560625], "NumGC":12635} Allocated: 3718694840, active: 3720273920, metadata: 21634640 (n_thp 0), resident: 3797233664, mapped: 3813122048, retained: 2040029184 2021-09-13 16:24:13 [ss-diag] 8 74694 73230 0 0 261 1 17108028 4 96181412 196780 1.000000 0.930642 1.000000 0:0:0 2021-09-13 16:24:13 [ss-diag] 9 159525 158240 0 0 501 81 32839548 208 97822840 13577804 1.000000 0.930642 1.000000 0:0:0 2021-09-13T16:24:55.638-07:00 [Info] memstats {"Alloc":338974840, "TotalAlloc":2138837723488, "Sys":976251128, "Lookups":0, "Mallocs":107203900,"Frees":107178182, "HeapAlloc":338974840, "HeapSys":934150144, "HeapIdle":592429056, "HeapInuse":341721088,"HeapReleased":408707072, "HeapObjects":25718,"GCSys":30994432, "LastGC":1631575495620481657,"PauseTotalNs":73794011331, "PauseNs":[967535, 3700186, 4664524, 98981283, 1355211, 3700774, 1324659, 1135141, 2031550, 1105743, 1042624, 909536, 41576836, 1279442, 1720109, 1559763, 5249528, 4671465, 2794508, 5664606, 746100, 780865, 1741817, 1613257, 4657672, 1009547, 1851845, 3323138, 1930665, 3957651, 1289693, 1364471, 703929, 32685617, 92146573, 1279847, 952897, 1226216, 4969533, 1452473, 1406121, 2412696, 1209370, 900023, 4548993, 3412676, 1577809, 916010, 3754545, 801237, 412039, 2450712, 2796044, 1614178, 167288073, 1627370, 1070787, 120596388, 2850000, 903184, 1293232, 1082160, 2022666, 3395501, 2978826, 5212341, 4604625, 1292416, 1131708, 2169877, 2047694, 4186648, 1033095, 1602748, 3940758, 818018, 1062038, 1727958, 6990866, 4650293, 1533592, 3796390, 31963680, 2155754, 1564150, 2280316, 3344161, 77116515, 1311999, 2212438, 1891448, 2627068, 1305161, 3349175, 3776630, 1299845, 3392064, 1085213, 1306942, 1048200, 3903495, 75360187, 1312685, 3042461, 1587573, 1741920, 43848011, 3101909, 1118427, 3058044, 4502842, 1015576, 3649319, 5324291, 92088957, 28745519, 74489404, 2423970, 980775, 1585339, 4385535, 1400495, 840195, 4045915, 1123652, 2369638, 6411565, 5649157, 1961480, 124256689, 871558, 3453364, 4309607, 3922540, 3469088, 2312874, 4319953, 602742, 1748870, 1758650, 3260365, 3688608, 4576090, 885279, 2832942, 1496831, 89129309, 181225396, 1681165, 2063212, 994452, 2019286, 5060070, 167110159, 982949, 155079279, 1654712, 2542183, 5369184, 63956084, 14564652, 136504917, 28874572, 904744, 923883, 2157922, 1373491, 892281, 2073698, 53748469, 2041159, 1451659, 1981253], "NumGC":12808} Allocated: 2439879488, active: 2441560064, metadata: 21641936 (n_thp 0), resident: 2593783808, mapped: 2609205248, retained: 3243945984 Mon Sep 13 16:25:14 PDT 2021 RSS = 3024040 2021-09-13 16:25:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 229, "num_active_sctxs": 44, "num_free_sctxs": 188, "num_trim_sctxs": 93, "num_wctxs": 286, "num_free_wctxs": 269, "num_active_wctxs": 17, "num_trim_wctxs": 652, "num_readers": 88, "num_writers": 98, "cpu": 15.68145, "total_core": 16, "threads": 54, "assigned_quota": 2567836927, "current_quota": 2567836927, "memory_resident": 0, "memory_jemalloc": 2557530112, "memory_jemalloc_allocated": 2449294264, "memory_in_use": 2570047488, "memory_stats_size_total": 2758407925, "memory_stats_size_global": 1029565400, "memory_stats_size_page": 960929488, "memory_stats_size_index": 68872904, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165648, "reclaim_pending": 3081532, "buf_memused": 1725524001, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 12, "num_active_smr": 7, "num_pending_smr": 1, "num_pages": 1060, "items_count": 2080, "total_records": 13606, "total_ops": 640504, "compacts": 879858, "purges": 0, "lss_data_size": 188654118, "lss_used_space": 689236973, "lss_fragmentation": 72, "lss_num_reads": 172252, "lss_read_bs": 43347072772, "lss_blk_read_bs": 44198408192, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 44002729984, "bytes_incoming": 24290122360, "write_amp_avg": 0.86, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 248 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 13606, "resident_ratio": 100.00000, "cleaner_num_reads": 172252, "cleaner_read_bs": 43347072772, "cleaner_blk_read_bs": 44198408192, "recovery_num_lssctx": 5, "recovery_num_sctxs": 28, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 8, "recovery_num_trim_sctxs": 8, "recovery_buf_memused": 3145728, "recovery_data_size": 48142564, "recovery_used_space": 196721964, "recovery_lss_blk_read_bs": 18882883584, "recovery_lss_read_bs": 18193908345, "recovery_lss__num_reads": 142099, "recovery_bytes_written": 18421755904, "recovery_lss_frag": 75, "recovery_cleaner_num_reads": 142099, "recovery_cleaner_read_bs": 18193908345, "recovery_cleaner_blk_read_bs": 18882883584, "mvcc_purge_ratio": 6.54135 }2021-09-13T16:25:26.125-07:00 [Info] memstats {"Alloc":453647168, "TotalAlloc":2179022143504, "Sys":976251128, "Lookups":0, "Mallocs":110552135,"Frees":110523541, "HeapAlloc":453647168, "HeapSys":934117376, "HeapIdle":477855744, "HeapInuse":456261632,"HeapReleased":412188672, "HeapObjects":28594,"GCSys":31027200, "LastGC":1631575526124027077,"PauseTotalNs":76278795174, "PauseNs":[1647983, 177528119, 1656214, 1768661, 4047716, 987590, 1770577, 1200847, 21244956, 3264308, 3379107, 1822371, 1462795, 2933733, 1642882, 1512450, 1435136, 142594110, 1664399, 796446, 742236, 2748818, 1516589, 1504304, 1596880, 107001859, 763093, 3564812, 1317576, 1292265, 1447230, 1394804, 2133147, 4661331, 864423, 1731615, 1842392, 2363293, 1435225, 832601, 1518471, 1837955, 1017141, 1339247, 1048574, 4566328, 1792287, 983484, 123451208, 1485688, 3771205, 1600119, 1798108, 5484795, 1660776, 2021131, 1284349, 1280323, 1611385, 577826, 2177138, 1210480, 1523218, 2414162, 1208251, 3253668, 818894, 2454812, 3629367, 1630797, 3531567, 928282, 5224101, 854440, 91401798, 959507, 817149, 173384787, 926997, 3225266, 1223869, 1523739, 4660305, 1969776, 3156994, 4645671, 1762503, 1795080, 1591134, 857999, 2054095, 3157006, 3254684, 1595706, 1717147, 1636573, 1187931, 1937712, 3921077, 5953007, 1023605, 996387, 3553029, 1515670, 1590085, 2997774, 2428849, 107811491, 1308941, 1779046, 2822139, 1585886, 2395084, 2164875, 2356439, 1883653, 541864, 1834141, 57050382, 1644282, 1462519, 885870, 1841576, 1878575, 1552211, 1747784, 130426080, 1481810, 1040079, 1490293, 3463112, 4601299, 2227113, 2255446, 67811561, 118210186, 1940915, 1995544, 4606816, 3737455, 51492927, 59897662, 96664297, 74142328, 2216713, 1748419, 2796735, 2519622, 1690231, 97643676, 1652685, 1842922, 114911428, 2659329, 4861403, 85733968, 148997410, 4214998, 1445938, 1137679, 827257, 5311737, 3700654, 3653430, 872587, 109179122, 22606838], "NumGC":12975} Allocated: 2495446584, active: 2497257472, metadata: 21641936 (n_thp 0), resident: 2633355264, mapped: 2649235456, retained: 3203915776 2021-09-13 16:25:26 [ss-workload] instance inse% dele% read% 2021-09-13 16:25:26 [ss-workload] 0 14% 14% 70% 2021-09-13 16:25:26 [ss-workload] 1 14% 14% 70% 2021-09-13 16:25:26 [ss-workload] 2 14% 14% 70% 2021-09-13 16:25:26 [ss-workload] 3 14% 14% 70% 2021-09-13 16:25:26 [ss-workload] 4 3% 3% 92% 2021-09-13 16:25:26 [ss-workload] 5 3% 3% 92% 2021-09-13 16:25:26 [ss-workload] 6 3% 3% 92% 2021-09-13 16:25:26 [ss-workload] 7 18% 18% 62% 2021-09-13 16:25:26 [ss-workload] 8 18% 18% 62% 2021-09-13 16:25:26 [ss-workload] 9 18% 18% 62% 2021-09-13 16:25:26 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:25:26 [ss-diag] 0 49028 47526 0 0 277 1 18156796 4 98676604 196812 1.000000 5.100208 1.000000 0:0:0 2021-09-13 16:25:26 [ss-diag] 1 48215 46719 0 0 277 1 18156796 4 98280996 196764 1.000000 8.327857 1.000000 0:0:0 2021-09-13 16:25:26 [ss-diag] 2 48305 46651 0 0 260 1 17042480 4 108645148 196780 1.000000 0.935979 1.000000 0:0:0 2021-09-13 16:25:26 [ss-diag] 3 47516 46194 0 0 257 1 16845836 4 86870124 196764 1.000000 0.935979 1.000000 0:0:0 2021-09-13 16:25:26 [ss-diag] 4 1834114 1832904 0 0 3849 664 252294252 208 92905084 13577836 1.000000 2.409645 1.000000 0:0:0 2021-09-13 16:25:26 [ss-diag] 5 1818746 1817566 0 0 4439 749 290967572 208 90936996 13578028 1.000000 0.935979 1.000000 0:0:0 2021-09-13 16:25:26 [ss-diag] 6 1862036 1860772 0 0 4252 726 278710096 208 96442932 13577692 1.000000 0.935979 1.000000 0:0:0 2021-09-13 16:25:26 [ss-diag] 7 185829 184557 0 0 347 57 22745156 208 96979540 13577644 1.000000 4.077999 1.000000 0:0:0 2021-09-13 16:25:26 [ss-diag] 8 78475 77091 0 0 249 1 16321452 4 90936292 196780 1.000000 0.935979 1.000000 0:0:0 2021-09-13 16:25:26 [ss-diag] 9 185902 184623 0 0 457 74 29955436 208 97429088 13577804 1.000000 0.935979 1.000000 0:0:0 2021-09-13 16:25:46 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: starting... frag 56, data: 166045485, used: 385636775 log:(22455625275 - 22841262080) head:22348298793 2021-09-13 16:25:46 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: completed... frag 81, data: 104493659, used: 580076044, disk: 583008816, relocated: 213954, retries: 16298, skipped: 8069, cleaned:4677741151, log:(27133366426 - 27714084864), disk:(27131076048-27714084864), activeFragRatio:30, activeMaxFragRatio:80, run:10 duration:291644 ms elapsed: 304875 ms 2021-09-13T16:25:56.386-07:00 [Info] memstats {"Alloc":448359184, "TotalAlloc":2220827127208, "Sys":976251128, "Lookups":0, "Mallocs":114037908,"Frees":114004705, "HeapAlloc":448359184, "HeapSys":934051840, "HeapIdle":483328000, "HeapInuse":450723840,"HeapReleased":407601152, "HeapObjects":33203,"GCSys":30896128, "LastGC":1631575556246898494,"PauseTotalNs":78220561167, "PauseNs":[1531122, 1402443, 427564, 154666047, 22484432, 4249485, 969294, 1930494, 4160184, 56908391, 3816821, 1903103, 128449833, 3919422, 2003033, 3733248, 2288018, 6205443, 65041529, 1800626, 1912036, 1350348, 1644859, 1512172, 1236100, 993776, 871442, 1107644, 687986, 1429259, 1497396, 1544468, 1108301, 1354582, 1133234, 3769394, 1745011, 2540758, 3000796, 2616508, 1420791, 1227239, 1569907, 1531032, 3220222, 1778863, 1756315, 1883897, 3785806, 3106079, 1569889, 84050795, 905722, 1335561, 1426401, 811087, 3456823, 4160739, 5010858, 882330, 30951482, 1986610, 2359857, 1931038, 2946598, 2139774, 1488311, 1878461, 1687204, 2016595, 3899487, 1988757, 1061221, 1943582, 1745983, 2091336, 3012094, 1394775, 3339115, 147779073, 107431332, 1783113, 2286348, 1490473, 616496, 1321037, 3095007, 877051, 993859, 2526459, 2598936, 1403911, 2136708, 1235399, 605658, 1731680, 379557, 5446223, 168508610, 1453374, 5626697, 1523926, 112514910, 2007054, 3016648, 665934, 4308144, 1167285, 1540619, 392120, 3668135, 1771867, 1067029, 1157234, 787253, 2124779, 1372761, 3465618, 1328389, 1142106, 1722673, 3208811, 1679876, 1151330, 4927397, 2244918, 1504872, 5440205, 5435228, 1169687, 1134383, 2068845, 1530701, 148310026, 3876122, 3275963, 189769527, 859318, 1982165, 1605730, 3272505, 1285503, 1636540, 1847579, 5806927, 1783909, 1442526, 3993253, 1292580, 1829555, 3345248, 2911123, 6149207, 814763, 2520366, 1782651, 51495171, 60115649, 2151042, 1596096, 3896169, 1543262, 962385, 1388012, 21574337, 1132985, 1078517, 47268748, 10328132, 869954, 5406680, 1473176, 2825616], "NumGC":13148} Allocated: 2466038840, active: 2467885056, metadata: 21654608 (n_thp 0), resident: 2579582976, mapped: 2594992128, retained: 3258159104 2021-09-13 16:25:57 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: starting... frag 49, data: 33190778, used: 65657605 log:(573494523 - 639152128) head:530749766 2021-09-13 16:25:57 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: completed... frag 36, data: 35575651, used: 56094921, disk: 69334772, relocated: 758, retries: 513, skipped: 83, cleaned:1030806076, log:(1604300599 - 1660395520), disk:(1591060748-1660395520), activeFragRatio:30, activeMaxFragRatio:80, run:18 duration:50321 ms elapsed: 309438 ms 2021-09-13 16:26:00 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: starting... frag 36, data: 31620757, used: 50006979 log:(2468102748 - 2518360064) head:2464528069 2021-09-13 16:26:00 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: completed... frag 30, data: 38506587, used: 55520567, disk: 61349761, relocated: 2962860, retries: 3600, skipped: 73556, cleaned:1337405190, log:(3805507938 - 3861278720), disk:(3799928959-3861278720), activeFragRatio:30, activeMaxFragRatio:80, run:260 duration:84751 ms elapsed: 298845 ms 2021-09-13 16:26:03 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 88, data: 5843648, used: 49376772 log:(21463142890 - 21513441280) head:21463142890 2021-09-13 16:26:03 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 95, data: 2106368, used: 44834658, disk: 45887418, relocated: 36835906, retries: 164239, skipped: 9526481, cleaned:9969905314, log:(31433048204 - 31477882880), disk:(31431995462-31477882880), activeFragRatio:30, activeMaxFragRatio:80, run:231 duration:287602 ms elapsed: 301011 ms 2021-09-13 16:26:07 [gsi-test] [] QuotaTuner: 2567836927 -> 2311053234 Mon Sep 13 16:26:14 PDT 2021 RSS = 3007768 2021-09-13 16:26:15 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 42, data: 17448812, used: 30154262 log:(18189840664 - 18220388352) head:18187735180 2021-09-13 16:26:15 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 52, data: 15350700, used: 32390434, disk: 33967988, relocated: 46446382, retries: 19412, skipped: 3410069, cleaned:9039265722, log:(27229106386 - 27262021632), disk:(27228053644-27262021632), activeFragRatio:30, activeMaxFragRatio:80, run:288 duration:257446 ms elapsed: 300457 ms 2021-09-13 16:26:17 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: starting... frag 98, data: 650101, used: 50195574 log:(10433681691 - 10484699136) head:10419753327 2021-09-13 16:26:17 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: completed... frag 89, data: 6149845, used: 58331019, disk: 79293051, relocated: 3355101, retries: 63895, skipped: 24934, cleaned:4275054426, log:(14708736117 - 14767067136), disk:(14687774085-14767067136), activeFragRatio:30, activeMaxFragRatio:80, run:80 duration:287088 ms elapsed: 287092 ms 2021-09-13 16:26:25 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: starting... frag 69, data: 33542016, used: 110149545 log:(393162551 - 503312384) head:380424023 2021-09-13 16:26:25 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: completed... frag 31, data: 46600738, used: 67803295, disk: 94594613, relocated: 773, retries: 523, skipped: 151, cleaned:1140721706, log:(1533884257 - 1601687552), disk:(1507092939-1601687552), activeFragRatio:30, activeMaxFragRatio:80, run:20 duration:53552 ms elapsed: 323750 ms 2021-09-13T16:26:26.762-07:00 [Info] memstats {"Alloc":326379632, "TotalAlloc":2261884393688, "Sys":976251128, "Lookups":0, "Mallocs":117410933,"Frees":117389132, "HeapAlloc":326379632, "HeapSys":934281216, "HeapIdle":604839936, "HeapInuse":329441280,"HeapReleased":421773312, "HeapObjects":21801,"GCSys":30896128, "LastGC":1631575586621094478,"PauseTotalNs":80124111375, "PauseNs":[2986272, 1162602, 104638867, 1849586, 892351, 4939301, 1930043, 3643315, 855660, 107320042, 3549520, 2925953, 1189408, 2893234, 5434131, 30533346, 1866352, 888589, 2741502, 2722624, 880240, 3876206, 1449525, 1147403, 2578075, 2566624, 1101586, 677391, 2779024, 5283748, 1162215, 915313, 1183367, 2009182, 1124648, 4177111, 458705, 2445463, 1210953, 1361218, 100830439, 1782783, 1402169, 1587431, 2314877, 1518144, 2003817, 3047827, 2062991, 1566218, 1370840, 1738038, 1875400, 4629493, 926139, 991027, 1965851, 1547089, 1197636, 2532657, 4555829, 140167385, 1715071, 609124, 1648409, 1367900, 1236494, 1073883, 3395914, 1574733, 53989928, 1582117, 2227103, 973109, 1932063, 4079621, 1526471, 1850866, 4734881, 432993, 1493258, 1043663, 809744, 1743975, 702522, 2802800, 1997867, 3585632, 4461500, 817540, 5238863, 1031755, 98070028, 69503412, 44558893, 11913838, 1264672, 1043450, 1480979, 1064070, 2054322, 1855101, 1726151, 2939325, 2055782, 1803392, 1965971, 1612611, 107686879, 3867741, 1605424, 4309189, 1420212, 2252069, 14287212, 2327381, 38010252, 1341296, 110398510, 2024525, 10144161, 681052, 1592417, 3418918, 121587646, 3315248, 1386420, 1205707, 56668532, 3622766, 927498, 3460156, 1303515, 1554936, 137868658, 1147246, 2485492, 1813330, 3754150, 1129087, 734494, 1222262, 1027987, 1550687, 1360817, 63477597, 70736808, 1286103, 2108385, 4188787, 935376, 3247618, 886810, 97085535, 7144536, 2229657, 2188089, 3378763, 2514256, 2539380, 1208949, 4113771, 3552437, 853740, 2488855, 1025646, 2124122, 1241192, 2833843, 1398914, 1769177, 941419], "NumGC":13320} Allocated: 2508864760, active: 2510774272, metadata: 21654608 (n_thp 0), resident: 2717761536, mapped: 2733629440, retained: 3119521792 2021-09-13 16:26:31 [gsi-test] [] Stop and verify... 2021-09-13 16:26:32 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper stopped 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:26:32 [gsi-test] [] { "memory_quota": 2311053234, "count": 20698, "compacts": 104, "purges": 0, "splits": 3, "merges": 0, "inserts": 20698, "deletes": 0, "compact_conflicts": 211, "split_conflicts": 10, "merge_conflicts": 0, "insert_conflicts": 2974, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 101562316, "memory_size_index": 196812, "allocated": 3342063048, "freed": 3240500732, "reclaimed": 3240500732, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 196812, "freed_index": 0, "reclaimed_index": 0, "num_pages": 4, "items_count": 208, "total_records": 1546, "num_rec_allocs": 50930, "num_rec_frees": 49384, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1356712504, "lss_data_size": 51244247, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 77642798, "est_recovery_size": 8038404, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 61869, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 41171, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.43269, "currSn": 1476, "gcSn": 1467, "gcSnIntervals": "[0 581 1144 1468]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 18, "num_trim_wctxs": 68, "num_readers": 11, "num_writers": 5, "page_bytes": 18025700, "page_cnt": 1, "page_itemcnt": 275, "avg_item_size": 65548, "avg_page_size": 18025700, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 92707792, "page_bytes_marshalled":1873832780, "page_bytes_compressed":1019926528, "compression_ratio": 1.83722, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 19840, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 124294597, "lss_data_size": 51645935, "lss_used_space": 78316076, "lss_disk_size": 142147584, "lss_fragmentation": 34, "lss_num_reads": 217, "lss_read_bs": 1906090602, "lss_blk_read_bs": 1907597312, "bytes_written": 1954086912, "bytes_incoming": 1356712504, "write_amp": 6.45, "write_amp_avg": 1.43, "lss_gc_num_reads": 217, "lss_gc_reads_bs": 1906090602, "lss_blk_gc_reads_bs": 1907597312, "lss_gc_status": "frag 20, data: 51244247, used: 64483884, disk: 90745919, relocated: 368, retries: 240, skipped: 25, cleaned:271470237, log:(1875770836 - 1940254720), disk:(1849508801-1940254720), activeFragRatio:30, activeMaxFragRatio:80, run:5 duration:13644 ms elapsed: 23249 ms", "lss_head_offset": 1875770836, "lss_tail_offset": 1940254720, "num_sctxs": 21, "num_free_sctxs": 13, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:26:32 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : All daemons stopped 2021-09-13 16:26:32 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:26:33 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper stopped 2021-09-13 16:26:33 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:26:33 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:26:33 [gsi-test] [] { "memory_quota": 2311053234, "count": 20302, "compacts": 102, "purges": 0, "splits": 3, "merges": 0, "inserts": 20302, "deletes": 0, "compact_conflicts": 167, "split_conflicts": 8, "merge_conflicts": 0, "insert_conflicts": 2672, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 3, "memory_size": 100775516, "memory_size_index": 196764, "allocated": 3262006012, "freed": 3161230496, "reclaimed": 3161230496, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 196764, "freed_index": 0, "reclaimed_index": 0, "num_pages": 4, "items_count": 208, "total_records": 1534, "num_rec_allocs": 49710, "num_rec_frees": 48176, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1330755496, "lss_data_size": 53670385, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 101264877, "est_recovery_size": 7235128, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 60593, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 40291, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.37500, "currSn": 1475, "gcSn": 1473, "gcSnIntervals": "[0 579 1155 1474]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 16, "num_trim_wctxs": 74, "num_readers": 8, "num_writers": 8, "page_bytes": 17435768, "page_cnt": 1, "page_itemcnt": 266, "avg_item_size": 65548, "avg_page_size": 17435768, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 101102592, "page_bytes_marshalled":1800420528, "page_bytes_compressed":979945307, "compression_ratio": 1.83727, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 19712, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 323036374, "lss_data_size": 54072073, "lss_used_space": 103555072, "lss_disk_size": 161005568, "lss_fragmentation": 47, "lss_num_reads": 202, "lss_read_bs": 1801636736, "lss_blk_read_bs": 1802047488, "bytes_written": 1771618304, "bytes_incoming": 1330755496, "write_amp": 3.89, "write_amp_avg": 1.32, "lss_gc_num_reads": 202, "lss_gc_reads_bs": 1801636736, "lss_blk_gc_reads_bs": 1802047488, "lss_gc_status": "frag 33, data: 52315569, used: 79121172 log:(1668060226 - 1747181568) head:1655674695", "lss_head_offset": 1668060226, "lss_tail_offset": 1758777344, "num_sctxs": 25, "num_free_sctxs": 16, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:26:34 [gsi-test] [] { "memory_quota": 2311053234, "count": 20197, "compacts": 101, "purges": 0, "splits": 3, "merges": 0, "inserts": 20197, "deletes": 0, "compact_conflicts": 197, "split_conflicts": 6, "merge_conflicts": 0, "insert_conflicts": 2655, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 108317720, "memory_size_index": 196780, "allocated": 3292814804, "freed": 3184497084, "reclaimed": 3184497084, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 196780, "freed_index": 0, "reclaimed_index": 0, "num_pages": 4, "items_count": 208, "total_records": 1649, "num_rec_allocs": 50179, "num_rec_frees": 48530, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1323872956, "lss_data_size": 48027871, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 208816830, "est_recovery_size": 401842, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 60414, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 40217, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.92788, "currSn": 1382, "gcSn": 1378, "gcSnIntervals": "[0 466 1047 1379]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 12, "num_trim_wctxs": 75, "num_readers": 9, "num_writers": 4, "page_bytes": 15534876, "page_cnt": 1, "page_itemcnt": 237, "avg_item_size": 65548, "avg_page_size": 15534876, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 95068960, "page_bytes_marshalled":2405433932, "page_bytes_compressed":1309266966, "compression_ratio": 1.83724, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 23920, "num_total_smr": 1, "num_active_smr": 3, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 428275810, "lss_data_size": 148042522, "lss_used_space": 647704166, "lss_disk_size": 728891392, "lss_fragmentation": 77, "lss_num_reads": 62048, "lss_read_bs": 31846058631, "lss_blk_read_bs": 32718241792, "bytes_written": 32337166336, "bytes_incoming": 28189441784, "write_amp": 0.92, "write_amp_avg": 1.01, "lss_gc_num_reads": 62048, "lss_gc_reads_bs": 31846058631, "lss_blk_gc_reads_bs": 32718241792, "lss_gc_status": "frag 68, data: 151435218, used: 487489916 log:(27713513627 - 28201574400) head:27711226681", "lss_head_offset": 27901438054, "lss_tail_offset": 28403851264, "num_sctxs": 53, "num_free_sctxs": 39, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:26:34 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : All daemons stopped 2021-09-13 16:26:34 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper stopped 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:26:34 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:26:34 [gsi-test] [] { "memory_quota": 2311053234, "count": 19798, "compacts": 101, "purges": 0, "splits": 3, "merges": 0, "inserts": 19798, "deletes": 0, "compact_conflicts": 223, "split_conflicts": 5, "merge_conflicts": 0, "insert_conflicts": 2936, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 4, "memory_size": 83590404, "memory_size_index": 196764, "allocated": 3237341772, "freed": 3153751368, "reclaimed": 3153751368, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 196764, "freed_index": 0, "reclaimed_index": 0, "num_pages": 4, "items_count": 208, "total_records": 1272, "num_rec_allocs": 49331, "num_rec_frees": 48059, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1297719304, "lss_data_size": 13817322, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 60075313, "est_recovery_size": 401842, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 59187, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 39389, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 6.11538, "currSn": 1374, "gcSn": 1371, "gcSnIntervals": "[0 468 1050 1372]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 13, "num_trim_wctxs": 77, "num_readers": 6, "num_writers": 5, "page_bytes": 17697960, "page_cnt": 1, "page_itemcnt": 270, "avg_item_size": 65548, "avg_page_size": 17697960, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 111072192, "page_bytes_marshalled":2419199346, "page_bytes_compressed":1316761342, "compression_ratio": 1.83723, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 8072, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 428275810, "lss_data_size": 146446969, "lss_used_space": 647704166, "lss_disk_size": 728891392, "lss_fragmentation": 77, "lss_num_reads": 62048, "lss_read_bs": 31846058631, "lss_blk_read_bs": 32718241792, "bytes_written": 32337166336, "bytes_incoming": 28197831928, "write_amp": 0.92, "write_amp_avg": 1.01, "lss_gc_num_reads": 62048, "lss_gc_reads_bs": 31846058631, "lss_blk_gc_reads_bs": 32718241792, "lss_gc_status": "frag 68, data: 151435218, used: 487489916 log:(27713513627 - 28201574400) head:27711226681", "lss_head_offset": 27901438054, "lss_tail_offset": 28403851264, "num_sctxs": 53, "num_free_sctxs": 44, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:26:34 [gsi-test] [] { "memory_quota": 2311053234, "count": 162834, "compacts": 325134, "purges": 0, "splits": 207, "merges": 0, "inserts": 162834, "deletes": 0, "compact_conflicts": 246, "split_conflicts": 28, "merge_conflicts": 0, "insert_conflicts": 2418, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 4, "memory_size": 89625812, "memory_size_index": 13577836, "allocated": 155548545628, "freed": 155458919816, "reclaimed": 155456166128, "reclaim_pending": 2753688, "reclaim_list_size": 2753688, "reclaim_list_count": 7, "reclaim_threshold": 1, "allocated_index": 13577836, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1160, "num_rec_allocs": 2048727, "num_rec_frees": 2047567, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 10673443032, "lss_data_size": 1935551, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 64518366, "est_recovery_size": 45252368, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 2620924, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 2458090, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 5.57692, "currSn": 26350, "gcSn": 26331, "gcSnIntervals": "[0 9651 20176 26332]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 5, "num_free_wctxs": 28, "num_trim_wctxs": 90, "num_readers": 12, "num_writers": 13, "page_bytes": 236497184, "page_cnt": 647, "page_itemcnt": 3608, "avg_item_size": 65548, "avg_page_size": 365528, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 104647288, "page_bytes_marshalled":22628633828, "page_bytes_compressed":12330794796, "compression_ratio": 1.83513, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12416, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 177346710, "lss_data_size": 2730599, "lss_used_space": 103131881, "lss_disk_size": 133382144, "lss_fragmentation": 97, "lss_num_reads": 302630, "lss_read_bs": 47211459108, "lss_blk_read_bs": 47835193344, "bytes_written": 47445131264, "bytes_incoming": 10673443032, "write_amp": 2.30, "write_amp_avg": 1.40, "lss_gc_num_reads": 302630, "lss_gc_reads_bs": 47211459108, "lss_blk_gc_reads_bs": 47835193344, "lss_gc_status": "frag 89, data: 6008495, used: 57191215, disk: 78234207, relocated: 248701, retries: 5357, skipped: 1580, cleaned:216214972, log:(14924951089 - 14982713344), disk:(14904479137-14982713344), activeFragRatio:30, activeMaxFragRatio:80, run:5 duration:15714 ms elapsed: 15714 ms", "lss_head_offset": 14924951089, "lss_tail_offset": 14982967296, "num_sctxs": 39, "num_free_sctxs": 30, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:26:34 [gsi-test] [] { "memory_quota": 2311053234, "count": 160718, "compacts": 318780, "purges": 0, "splits": 207, "merges": 0, "inserts": 160718, "deletes": 0, "compact_conflicts": 280, "split_conflicts": 21, "merge_conflicts": 0, "insert_conflicts": 2196, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 10, "memory_size": 92510020, "memory_size_index": 13578028, "allocated": 154270809548, "freed": 154178299528, "reclaimed": 154174955772, "reclaim_pending": 3343756, "reclaim_list_size": 917880, "reclaim_list_count": 2, "reclaim_threshold": 1, "allocated_index": 13578028, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1204, "num_rec_allocs": 2035516, "num_rec_frees": 2034312, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 10534743464, "lss_data_size": 2364094, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 10278669, "est_recovery_size": 54634050, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 2634658, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 2473940, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 5.78846, "currSn": 25738, "gcSn": 25719, "gcSnIntervals": "[0 7887 19143 25720]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 25, "num_trim_wctxs": 93, "num_readers": 13, "num_writers": 11, "page_bytes": 261012136, "page_cnt": 674, "page_itemcnt": 3982, "avg_item_size": 65548, "avg_page_size": 387258, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 105563792, "page_bytes_marshalled":5162940240, "page_bytes_compressed":2813124221, "compression_ratio": 1.83530, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12160, "num_total_smr": 3, "num_active_smr": 1, "num_pending_smr": 1, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 428275810, "lss_data_size": 146446969, "lss_used_space": 647704166, "lss_disk_size": 728891392, "lss_fragmentation": 77, "lss_num_reads": 62048, "lss_read_bs": 31846058631, "lss_blk_read_bs": 32718241792, "bytes_written": 32337166336, "bytes_incoming": 28197831928, "write_amp": 0.92, "write_amp_avg": 1.01, "lss_gc_num_reads": 62048, "lss_gc_reads_bs": 31846058631, "lss_blk_gc_reads_bs": 32718241792, "lss_gc_status": "frag 68, data: 151435218, used: 487489916 log:(27713513627 - 28201574400) head:27711226681", "lss_head_offset": 27901438054, "lss_tail_offset": 28403851264, "num_sctxs": 53, "num_free_sctxs": 43, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:26:34 [gsi-test] [] { "memory_quota": 2311053234, "count": 161264, "compacts": 323492, "purges": 0, "splits": 207, "merges": 0, "inserts": 161264, "deletes": 0, "compact_conflicts": 298, "split_conflicts": 39, "merge_conflicts": 0, "insert_conflicts": 2406, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 9, "memory_size": 91854268, "memory_size_index": 13577692, "allocated": 156686381060, "freed": 156594526792, "reclaimed": 156591117496, "reclaim_pending": 3409296, "reclaim_list_size": 3409296, "reclaim_list_count": 8, "reclaim_threshold": 1, "allocated_index": 13577692, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1194, "num_rec_allocs": 2067617, "num_rec_frees": 2066423, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 10570532672, "lss_data_size": 1614396, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 7019113, "est_recovery_size": 53060202, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 2654600, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 2493336, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 5.74038, "currSn": 25535, "gcSn": 25519, "gcSnIntervals": "[0 7609 18901 25520]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 21, "num_trim_wctxs": 100, "num_readers": 11, "num_writers": 12, "page_bytes": 242396504, "page_cnt": 624, "page_itemcnt": 3698, "avg_item_size": 65548, "avg_page_size": 388455, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 104777040, "page_bytes_marshalled":4984383526, "page_bytes_compressed":2715850312, "compression_ratio": 1.83529, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 11968, "num_total_smr": 1, "num_active_smr": 3, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 428275810, "lss_data_size": 146066301, "lss_used_space": 647704166, "lss_disk_size": 728891392, "lss_fragmentation": 77, "lss_num_reads": 62048, "lss_read_bs": 31846058631, "lss_blk_read_bs": 32718241792, "bytes_written": 32337166336, "bytes_incoming": 28197831928, "write_amp": 0.92, "write_amp_avg": 1.01, "lss_gc_num_reads": 62048, "lss_gc_reads_bs": 31846058631, "lss_blk_gc_reads_bs": 32718241792, "lss_gc_status": "frag 68, data: 151435218, used: 487489916 log:(27713513627 - 28201574400) head:27711226681", "lss_head_offset": 27901438054, "lss_tail_offset": 28403851264, "num_sctxs": 53, "num_free_sctxs": 41, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:26:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:26:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:26:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:26:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:26:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:26:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:26:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:26:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:26:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:26:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:26:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:26:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:26:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:26:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:26:36 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : All daemons stopped 2021-09-13 16:26:36 [gsi-test] [0] Stop and verify... 2021-09-13 16:26:36 [gsi-test] [] { "memory_quota": 2311053234, "count": 35256, "compacts": 27210, "purges": 0, "splits": 207, "merges": 0, "inserts": 35256, "deletes": 0, "compact_conflicts": 305, "split_conflicts": 30, "merge_conflicts": 0, "insert_conflicts": 4181, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 3, "memory_size": 98156220, "memory_size_index": 13577644, "allocated": 15349466796, "freed": 15251310576, "reclaimed": 15250392504, "reclaim_pending": 918072, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577644, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1290, "num_rec_allocs": 206671, "num_rec_frees": 205381, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 2310960288, "lss_data_size": 26647566, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 64994063, "est_recovery_size": 33318400, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 95903, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 60647, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 6.20192, "currSn": 1704, "gcSn": 1701, "gcSnIntervals": "[0 665 1308 1702]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 14, "num_trim_wctxs": 73, "num_readers": 3, "num_writers": 9, "page_bytes": 34412700, "page_cnt": 83, "page_itemcnt": 525, "avg_item_size": 65548, "avg_page_size": 414610, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 114225248, "page_bytes_marshalled":7104321470, "page_bytes_compressed":3870981967, "compression_ratio": 1.83528, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 20544, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 64454699, "lss_data_size": 38457702, "lss_used_space": 94514312, "lss_disk_size": 165105664, "lss_fragmentation": 59, "lss_num_reads": 231087, "lss_read_bs": 31633529520, "lss_blk_read_bs": 32602210304, "bytes_written": 31840489472, "bytes_incoming": 2310960288, "write_amp": 3.47, "write_amp_avg": 1.74, "lss_gc_num_reads": 231087, "lss_gc_reads_bs": 31633529520, "lss_blk_gc_reads_bs": 32602210304, "lss_gc_status": "frag 40, data: 36289997, used: 61201614, disk: 69888121, relocated: 391454, retries: 532, skipped: 7924, cleaned:161606790, log:(3967114728 - 4028637184), disk:(3958749063-4028637184), activeFragRatio:30, activeMaxFragRatio:80, run:28 duration:10910 ms elapsed: 31865 ms", "lss_head_offset": 3967114728, "lss_tail_offset": 4028637184, "num_sctxs": 26, "num_free_sctxs": 18, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:26:38 [gsi-test] [0] Stop and verify... OK 2021-09-13 16:26:38 [gsi-test] [1] Stop and verify... 2021-09-13 16:26:39 [gsi-test] [] { "memory_quota": 2311053234, "count": 35080, "compacts": 175, "purges": 0, "splits": 3, "merges": 0, "inserts": 35080, "deletes": 0, "compact_conflicts": 421, "split_conflicts": 14, "merge_conflicts": 0, "insert_conflicts": 5577, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 5, "memory_size": 113891196, "memory_size_index": 196780, "allocated": 5327564864, "freed": 5213673668, "reclaimed": 5213673668, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 196780, "freed_index": 0, "reclaimed_index": 0, "num_pages": 4, "items_count": 206, "total_records": 1734, "num_rec_allocs": 81164, "num_rec_frees": 79430, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 2299423840, "lss_data_size": 40891144, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 177787582, "est_recovery_size": 598602, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 92432, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 57352, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 8.41748, "currSn": 1575, "gcSn": 1566, "gcSnIntervals": "[0 508 1170 1567]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 10, "num_trim_wctxs": 67, "num_readers": 3, "num_writers": 7, "page_bytes": 16583644, "page_cnt": 1, "page_itemcnt": 253, "avg_item_size": 65548, "avg_page_size": 16583644, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 103724816, "page_bytes_marshalled":2736520586, "page_bytes_compressed":1489491924, "compression_ratio": 1.83722, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 25440, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 428275810, "lss_data_size": 142110008, "lss_used_space": 645786385, "lss_disk_size": 771510272, "lss_fragmentation": 77, "lss_num_reads": 62125, "lss_read_bs": 31856157181, "lss_blk_read_bs": 32728506368, "bytes_written": 32379785216, "bytes_incoming": 28291565568, "write_amp": 0.92, "write_amp_avg": 1.01, "lss_gc_num_reads": 62125, "lss_gc_reads_bs": 31856157181, "lss_blk_gc_reads_bs": 32728506368, "lss_gc_status": "frag 72, data: 139654815, used: 510499929, disk: 545032090, relocated: 53551, retries: 3686, skipped: 1622, cleaned:802603789, log:(27935970215 - 28446470144), disk:(27901438054-28446470144), activeFragRatio:30, activeMaxFragRatio:80, run:2 duration:49575 ms elapsed: 49575 ms", "lss_head_offset": 27935970215, "lss_tail_offset": 28446470144, "num_sctxs": 53, "num_free_sctxs": 42, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:26:39 [gsi-test] [] { "memory_quota": 2311053234, "count": 33250, "compacts": 27027, "purges": 0, "splits": 207, "merges": 0, "inserts": 33250, "deletes": 0, "compact_conflicts": 311, "split_conflicts": 32, "merge_conflicts": 0, "insert_conflicts": 3957, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 1, "memory_size": 97628892, "memory_size_index": 13577804, "allocated": 15130513900, "freed": 15032885008, "reclaimed": 15029147708, "reclaim_pending": 3737300, "reclaim_list_size": 1835952, "reclaim_list_count": 4, "reclaim_threshold": 1, "allocated_index": 13577804, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1282, "num_rec_allocs": 203558, "num_rec_frees": 202276, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 2179471000, "lss_data_size": 26331965, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 114486804, "est_recovery_size": 30239406, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 91556, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 58306, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 6.16346, "currSn": 1588, "gcSn": 1585, "gcSnIntervals": "[0 504 1176 1586]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 14, "num_trim_wctxs": 92, "num_readers": 4, "num_writers": 9, "page_bytes": 19336660, "page_cnt": 48, "page_itemcnt": 295, "avg_item_size": 65548, "avg_page_size": 402847, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 112384016, "page_bytes_marshalled":3801150942, "page_bytes_compressed":2071140298, "compression_ratio": 1.83529, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 17664, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 428275810, "lss_data_size": 142110008, "lss_used_space": 645786385, "lss_disk_size": 771510272, "lss_fragmentation": 77, "lss_num_reads": 62125, "lss_read_bs": 31856157181, "lss_blk_read_bs": 32728506368, "bytes_written": 32379785216, "bytes_incoming": 28291631116, "write_amp": 0.92, "write_amp_avg": 1.01, "lss_gc_num_reads": 62125, "lss_gc_reads_bs": 31856157181, "lss_blk_gc_reads_bs": 32728506368, "lss_gc_status": "frag 72, data: 139654815, used: 510499929, disk: 545032090, relocated: 53551, retries: 3686, skipped: 1622, cleaned:802603789, log:(27935970215 - 28446470144), disk:(27901438054-28446470144), activeFragRatio:30, activeMaxFragRatio:80, run:2 duration:49575 ms elapsed: 49575 ms", "lss_head_offset": 27935970215, "lss_tail_offset": 28446470144, "num_sctxs": 53, "num_free_sctxs": 42, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:26:39 [gsi-test] [1] Stop and verify... OK 2021-09-13 16:26:39 [gsi-test] [2] Stop and verify... 2021-09-13 16:26:40 [gsi-test] [2] Stop and verify... OK 2021-09-13 16:26:40 [gsi-test] [3] Stop and verify... 2021-09-13 16:26:42 [gsi-test] [3] Stop and verify... OK 2021-09-13 16:26:42 [gsi-test] [4] Stop and verify... 2021-09-13 16:26:42 [gsi-test] [4] Stop and verify... OK 2021-09-13 16:26:42 [gsi-test] [5] Stop and verify... 2021-09-13 16:26:42 [gsi-test] [5] Stop and verify... OK 2021-09-13 16:26:42 [gsi-test] [6] Stop and verify... 2021-09-13 16:26:42 [gsi-test] [6] Stop and verify... OK 2021-09-13 16:26:42 [gsi-test] [7] Stop and verify... 2021-09-13 16:26:42 [ss-workload] instance inse% dele% read% 2021-09-13 16:26:42 [ss-workload] 0 14% 14% 70% 2021-09-13 16:26:43 [gsi-test] [7] Stop and verify... OK 2021-09-13 16:26:43 [gsi-test] [8] Stop and verify... 2021-09-13 16:26:43 [gsi-test] [8] Stop and verify... OK 2021-09-13 16:26:43 [gsi-test] [9] Stop and verify... 2021-09-13 16:26:42 [ss-workload] 1 14% 14% 70% 2021-09-13 16:26:42 [ss-workload] 2 14% 14% 70% 2021-09-13 16:26:44 [gsi-test] [9] Stop and verify... OK 2021-09-13 16:26:44 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper started 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : all deamons started 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : all deamons started 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : all deamons started 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:26:44 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper started 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : all deamons started 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : all deamons started 2021-09-13 16:26:44 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper started 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : all deamons started 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : all deamons started 2021-09-13 16:26:44 [gsi-test] [] Stop and verify... OK 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:26:44 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:26:42 [ss-workload] 3 14% 14% 70% 2021-09-13 16:26:42 [ss-workload] 4 3% 3% 92% 2021-09-13 16:26:42 [ss-workload] 5 3% 3% 92% 2021-09-13 16:26:42 [ss-workload] 6 3% 3% 92% 2021-09-13 16:26:42 [ss-workload] 7 18% 18% 62% 2021-09-13 16:26:42 [ss-workload] 8 18% 18% 62% 2021-09-13 16:26:42 [ss-workload] 9 18% 18% 62% 2021-09-13 16:26:42 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:26:42 [ss-diag] 0 51399 49834 0 0 268 1 17566864 4 102807944 196812 1.000000 6.449485 1.000000 0:0:0 2021-09-13 16:26:42 [ss-diag] 1 50498 49110 0 0 266 1 17435768 4 91198868 196764 1.000000 3.888056 1.000000 0:0:0 2021-09-13 16:26:42 [ss-diag] 2 51226 49910 0 0 237 1 15534876 4 86476532 196780 1.000000 0.921702 1.000000 0:0:0 2021-09-13 16:26:42 [ss-diag] 3 49788 48512 0 0 270 1 17697960 4 83852660 196764 1.000000 0.921702 1.000000 0:0:0 2021-09-13 16:26:42 [ss-diag] 4 2130543 2129367 0 0 7589 1304 497443772 208 90676436 13577836 1.000000 2.299613 1.000000 0:0:0 2021-09-13 16:26:42 [ss-diag] 5 2109312 2108152 0 0 6864 1170 449921472 208 89625044 13578028 1.000000 0.921702 1.000000 0:0:0 2021-09-13 16:26:42 [ss-diag] 6 2150256 2149062 0 0 7776 1285 509701248 208 91854204 13577692 1.000000 0.921702 1.000000 0:0:0 2021-09-13 16:26:42 [ss-diag] 7 213137 211817 0 0 423 67 27726804 208 100124500 13577644 1.000000 3.466171 1.000000 0:0:0 2021-09-13 16:26:42 [ss-diag] 8 82393 80887 0 0 253 1 16583644 4 98937580 196780 1.000000 0.921702 1.000000 0:0:0 2021-09-13T16:26:57.257-07:00 [Info] memstats {"Alloc":495345864, "TotalAlloc":2304994947336, "Sys":976251128, "Lookups":0, "Mallocs":120558507,"Frees":120532499, "HeapAlloc":495345864, "HeapSys":934641664, "HeapIdle":436494336, "HeapInuse":498147328,"HeapReleased":424558592, "HeapObjects":26008,"GCSys":30994432, "LastGC":1631575617231566887,"PauseTotalNs":82878970916, "PauseNs":[1453854, 1620760, 648158, 2471349, 1538757, 2558352, 55983079, 73166570, 64394052, 2478105, 1266461, 3805552, 3447384, 1774081, 171786626, 2069356, 1113081, 1661436, 1139647, 1673366, 2076657, 4706457, 24882255, 1253299, 2594968, 1092320, 2737341, 1261089, 1971570, 285417490, 3820385, 1402376, 1241218, 1916279, 1398243, 1998376, 1648727, 1388051, 164839245, 43302115, 1156642, 765848, 1277460, 1232412, 919849, 797688, 1329472, 1914173, 1619467, 1587692, 1128730, 1761461, 1917172, 1448274, 1350684, 1473175, 1586413, 1056195, 1659389, 2053304, 1801033, 1518464, 1292934, 947846, 912093, 1483787, 1789489, 1645837, 2056475, 1394955, 771325, 1532939, 1751056, 1510930, 965429, 1497966, 1124736, 1699122, 1825773, 854656, 1279313, 1299997, 1621402, 1641025, 586330, 1724835, 1730975, 1125291, 1664378, 1812977, 744754, 1321398, 1422639, 930435, 1445783, 898329, 1476813, 1919660, 1132114, 1364678, 1249585, 1478651, 1515899, 1669279, 912370, 1497726, 720885, 910942, 1856234, 1479750, 1678795, 1712855, 1774151, 1736645, 1231547, 1369607, 1720322, 1306734, 1159305, 1722987, 1298177, 799987, 1798834, 900745, 2845406, 85593877, 2548068, 1355186, 1652337, 1432750, 275598674, 1352691, 1422872, 1325795, 2117743, 4192582, 4936819, 150682457, 6754107, 1023441, 1814461, 1356303, 3124143, 7335537, 1369506, 1161623, 118461906, 51953633, 123060343, 3389154, 1606285, 1519450, 1715864, 830078, 2065923, 16898045, 1073805, 1922511, 4662272, 180555753, 1639666, 107137844, 2540629, 440683, 3605997, 1622263, 3936359, 2547380, 151081534, 5113911, 1337704, 1825169, 1299391, 245673476, 1863128, 3574903, 1447389, 1688083, 3410338, 78306324], "NumGC":13500} Allocated: 2541082600, active: 2543009792, metadata: 21658576 (n_thp 0), resident: 2617503744, mapped: 2633564160, retained: 3219587072 2021-09-13 16:26:42 [ss-diag] 9 208755 207419 0 0 233 37 15272684 208 101165524 13577804 1.000000 0.921702 1.000000 0:0:0 Mon Sep 13 16:27:14 PDT 2021 RSS = 3029944 2021-09-13T16:27:27.632-07:00 [Info] memstats {"Alloc":488272928, "TotalAlloc":2346787055696, "Sys":976251128, "Lookups":0, "Mallocs":123853252,"Frees":123825637, "HeapAlloc":488272928, "HeapSys":934150144, "HeapIdle":443187200, "HeapInuse":490962944,"HeapReleased":332349440, "HeapObjects":27615,"GCSys":30994432, "LastGC":1631575647630462267,"PauseTotalNs":85258694186, "PauseNs":[1520100, 921079, 4065350, 145262683, 3419080, 2707326, 855018, 1323103, 4566600, 8136721, 1144554, 1382810, 1193861, 830783, 90129046, 114196988, 3428616, 689413, 2432123, 1124400, 3722492, 139453101, 2346845, 1753931, 4649077, 6341425, 1006723, 743454, 883701, 71213048, 862433, 1306869, 109837284, 1157629, 4205289, 2107903, 2724535, 157229242, 2047255, 1194848, 3290790, 1732656, 1208549, 3362749, 2462104, 3659330, 906090, 1035237, 1287392, 3751394, 3776524, 743685, 1052500, 3330135, 2911564, 997364, 2458407, 1003219, 3393629, 1494067, 2711349, 53735731, 2010323, 1227803, 209062486, 1875357, 1674674, 3303143, 1216616, 1032727, 1258110, 3226281, 932344, 1442338, 3619535, 5888988, 1844969, 1100829, 2117987, 1363897, 1467688, 4500173, 4086420, 1129592, 1800374, 2911381, 951383, 1190223, 1295427, 789680, 1502431, 148594210, 1973924, 3403415, 3351453, 1230278, 1223578, 1064116, 768085, 3280892, 2581288, 1576531, 2828536, 3723381, 1941664, 1310221, 975706, 1504024, 2834448, 3303237, 3526906, 1147198, 779448, 1652517, 177704887, 7981445, 1205924, 2621539, 4379433, 1478365, 2765844, 10359017, 1378096, 1422115, 3516768, 5127230, 4817171, 2174975, 111518198, 10651306, 24313230, 1352558, 1916319, 1038947, 1857823, 1400819, 1830454, 4020238, 3963488, 1706219, 1002372, 1201311, 5039610, 2109492, 5193238, 2260142, 1660957, 4672623, 66277957, 4118707, 1239264, 910187, 751836, 1231005, 1049607, 1563310, 774823, 973779, 2066782, 7390718, 1433420, 3246600, 2063629, 1517672, 1941070, 803703, 78219282, 56410371, 5108088, 581885, 2384749, 797919, 115385043, 141020273], "NumGC":13674} Allocated: 2484503904, active: 2486435840, metadata: 21658576 (n_thp 0), resident: 2623700992, mapped: 2639106048, retained: 3214045184 2021-09-13 16:27:37 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: starting... frag 87, data: 17817976, used: 144502210 log:(2875461878 - 3020488704) head:2875461878 2021-09-13 16:27:37 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: completed... frag 96, data: 5163792, used: 129425326, disk: 138251714, relocated: 0, retries: 0, skipped: 7321993, cleaned:1112489902, log:(3987951780 - 4117774336), disk:(3979522622-4117774336), activeFragRatio:30, activeMaxFragRatio:80, run:291 duration:3366 ms elapsed: 299832 ms 2021-09-13 16:27:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoveryCleaner: starting... frag 97, data: 401688, used: 16928768 log:(0 - 16928768) head:0 2021-09-13 16:27:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoveryCleaner: completed... frag 97, data: 401688, used: 16322299, disk: 16928768, relocated: 0, retries: 0, skipped: 4, cleaned:0, log:(0 - 16928768), disk:(0-16928768), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:7 ms elapsed: 7 ms 2021-09-13T16:27:57.841-07:00 [Info] memstats {"Alloc":348627600, "TotalAlloc":2386784143840, "Sys":976251128, "Lookups":0, "Mallocs":126992455,"Frees":126970400, "HeapAlloc":348627600, "HeapSys":934051840, "HeapIdle":582328320, "HeapInuse":351723520,"HeapReleased":422428672, "HeapObjects":22055,"GCSys":30896128, "LastGC":1631575677752888215,"PauseTotalNs":87725165380, "PauseNs":[702690, 165362402, 1371279, 1064140, 2159718, 2148129, 763574, 3425010, 2733458, 657996, 1548870, 1354109, 5810771, 3959850, 764643, 2880185, 1816321, 1716748, 4530425, 1199067, 1743172, 1414638, 1234118, 1781731, 2313476, 4777394, 2230876, 2712270, 2766329, 62424230, 77429372, 1955727, 2056011, 6511300, 2301795, 1540363, 1729513, 1393036, 1998232, 1316744, 827414, 793185, 869459, 1070743, 4518451, 2742733, 975601, 5392333, 1810223, 3246028, 3728976, 2767741, 354115, 4149509, 1579875, 1105042, 1873697, 1324571, 1262270, 1671180, 71948492, 4265044, 5327885, 1564432, 978852, 1424303, 4311653, 2351970, 722291, 3836173, 4147332, 2539248, 1998310, 938045, 3819098, 2758235, 2468368, 1629691, 1921723, 2805366, 1520741, 3394229, 62820616, 1436260, 1369329, 1759922, 40620579, 1531037, 1496210, 4356757, 920893, 1290756, 1271470, 1786901, 2992010, 3501867, 193277772, 4324897, 4963031, 51272812, 983130, 1744759, 1458872, 2175624, 1138943, 968156, 2545347, 1672496, 18273427, 126610214, 1151573, 1120159, 4254249, 4256167, 954362, 939367, 2930742, 53969677, 4783852, 771577, 150692516, 1725543, 1964520, 1542951, 1063008, 844199, 2561492, 1750467, 49456354, 1091269, 1827958, 5112418, 1237504, 1673613, 117583259, 3994120, 2288726, 1998954, 1682326, 1739425, 64843431, 1854165, 1564627, 1293917, 721842, 64458537, 85053912, 1128501, 3491956, 1525311, 2237227, 341959170, 1351596, 1535793, 2693786, 2338674, 1400312, 2183088, 147627893, 3627934, 888757, 2661775, 4361730, 2964313, 196415753, 3222811, 1203610], "NumGC":13841} Allocated: 2413279936, active: 2415230976, metadata: 21662544 (n_thp 0), resident: 2559594496, mapped: 2574995456, retained: 3278155776 2021-09-13 16:27:58 [ss-workload] instance inse% dele% read% 2021-09-13 16:27:58 [ss-workload] 0 14% 14% 70% 2021-09-13 16:27:58 [ss-workload] 1 14% 14% 70% 2021-09-13 16:27:58 [ss-workload] 2 14% 14% 70% 2021-09-13 16:27:58 [ss-workload] 3 14% 14% 70% 2021-09-13 16:27:58 [ss-workload] 4 3% 3% 92% 2021-09-13 16:27:58 [ss-workload] 5 3% 3% 92% 2021-09-13 16:27:58 [ss-workload] 6 3% 3% 92% 2021-09-13 16:27:58 [ss-workload] 7 18% 18% 62% 2021-09-13 16:27:58 [ss-workload] 8 18% 18% 62% 2021-09-13 16:27:58 [ss-workload] 9 18% 18% 62% 2021-09-13 16:27:58 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:27:58 [ss-diag] 0 53976 52660 0 0 533 2 34937084 4 86476468 196812 1.000000 3.856623 1.000000 0:0:0 2021-09-13 16:28:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 97, data: 401688, used: 16998400 log:(0 - 16998400) head:0 2021-09-13 16:28:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 97, data: 401688, used: 16125894, disk: 16998400, relocated: 0, retries: 0, skipped: 4, cleaned:0, log:(0 - 16998400), disk:(0-16998400), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:4 ms elapsed: 4 ms 2021-09-13 16:27:58 [ss-diag] 1 52556 50980 0 0 276 1 18091248 4 103530212 196764 1.000000 7.054770 1.000000 0:0:0 2021-09-13 16:27:58 [ss-diag] 2 53564 52264 0 0 248 1 16255904 4 85426932 196780 1.000000 0.897152 1.000000 0:0:0 2021-09-13 16:27:58 [ss-diag] 3 51688 50386 0 0 269 1 17632412 4 85557884 196764 1.000000 0.897152 1.000000 0:0:0 2021-09-13 16:27:58 [ss-diag] 4 2384749 2383553 0 0 3669 650 240495612 208 91987684 13577836 1.000000 2.390691 1.000000 0:0:0 2021-09-13 16:27:58 [ss-diag] 5 2394699 2393513 0 0 3625 629 237611500 208 91329948 13578028 1.000000 0.897152 1.000000 0:0:0 2021-09-13 16:27:58 [ss-diag] 6 2418929 2417737 0 0 3596 628 235710608 208 91722836 13577692 1.000000 0.897152 1.000000 0:0:0 2021-09-13 16:27:58 [ss-diag] 7 240534 239234 0 0 410 68 26874680 208 98815812 13577644 1.000000 3.669010 1.000000 0:0:0 2021-09-13 16:27:58 [ss-diag] 8 86185 84673 0 0 271 1 17763508 4 99331492 196780 1.000000 0.897152 1.000000 0:0:0 2021-09-13 16:27:58 [ss-diag] 9 233990 232706 0 0 341 53 22351868 208 97755716 13577804 1.000000 0.897152 1.000000 0:0:0 Mon Sep 13 16:28:14 PDT 2021 RSS = 2970528 2021-09-13T16:28:28.145-07:00 [Info] memstats {"Alloc":345393112, "TotalAlloc":2422940320792, "Sys":976251128, "Lookups":0, "Mallocs":129871066,"Frees":129846571, "HeapAlloc":345393112, "HeapSys":934051840, "HeapIdle":585654272, "HeapInuse":348397568,"HeapReleased":320741376, "HeapObjects":24495,"GCSys":30896128, "LastGC":1631575708068162154,"PauseTotalNs":91353259684, "PauseNs":[135901005, 44546252, 34763800, 1143483, 1651553, 187687358, 2104833, 2081276, 4147100, 4755489, 1119698, 6871960, 111713522, 16904729, 1270999, 5805556, 2919903, 2152163, 3360425, 1122906, 1144207, 43897708, 2412823, 35896354, 103019168, 1121776, 1256680, 1968185, 89524068, 60429641, 1688258, 1610063, 631574, 1168448, 2063909, 1366137, 1603168, 810831, 2884010, 3348598, 1486880, 1251414, 2209104, 1020612, 1447378, 1164102, 4662219, 3904857, 1056946, 3627409, 160892170, 135283469, 1443855, 1794642, 99527615, 11222094, 1353867, 2064991, 1390205, 1783184, 1969045, 2477505, 104005948, 1570800, 907500, 21258104, 2568106, 2656475, 4571084, 1058527, 203509911, 96055920, 803046, 1605781, 1247336, 5957163, 1420459, 154433054, 4933878, 4160379, 1297870, 4180936, 3197126, 1766091, 1468689, 4363429, 4009755, 1428664, 93996345, 2346575, 2045639, 124363696, 1016156, 1155352, 3089163, 3488519, 1096256, 3902094, 2844445, 172738334, 3093912, 1797083, 4948811, 157577343, 86118356, 1792152, 1737051, 232491785, 1201903, 2136952, 1485883, 3362769, 7389312, 3199859, 1137861, 1123467, 3285322, 3527795, 63901284, 6923314, 744636, 2800584, 4578836, 3517552, 26569132, 3224037, 4590492, 2067331, 2420475, 2710019, 698059, 91487193, 1419240, 5078594, 200046189, 3771052, 4114699, 2619172, 1210488, 1344289, 52014250, 1080601, 1245219, 1612755, 1364490, 4113391, 125119210, 1184639, 913035, 2598097, 64182550], "NumGC":13992} Allocated: 2495833912, active: 2497753088, metadata: 21662544 (n_thp 0), resident: 2620780544, mapped: 2636181504, retained: 3216969728 2021-09-13T16:28:58.424-07:00 [Info] memstats {"Alloc":388279608, "TotalAlloc":2462694634960, "Sys":976251128, "Lookups":0, "Mallocs":133017301,"Frees":132991132, "HeapAlloc":388279608, "HeapSys":934248448, "HeapIdle":543129600, "HeapInuse":391118848,"HeapReleased":400252928, "HeapObjects":26169,"GCSys":30896128, "LastGC":1631575738322523152,"PauseTotalNs":94038264329, "PauseNs":[2652377, 1830885, 1897231, 3803023, 6176752, 2424635, 19759170, 1709348, 3121244, 4907187, 3706728, 82257178, 133840358, 1737011, 1449431, 1417460, 1002098, 7251160, 1275465, 3725771, 1039305, 1044686, 2687364, 62795902, 1441136, 5514107, 3483739, 2067178, 56226467, 172023347, 26916465, 2750840, 575993, 1054039, 711044, 1234029, 1904104, 1590325, 8904518, 1243161, 122487231, 1027089, 1797084, 856794, 4039266, 1960748, 1730379, 1448388, 2499698, 1514742, 52682954, 1466839, 4384198, 117674692, 2128329, 1527264, 1087638, 1878642, 1182008, 1484815, 1481170, 4611521, 2009071, 4580962, 2302772, 1897811, 1160991, 513108, 3677360, 861132, 1182018, 1160476, 2212099, 4536122, 2206041, 601553, 946585, 1986441, 3472616, 3014663, 1633571, 84250453, 52764744, 1609455, 3291300, 4346881, 2564886, 1528815, 773083, 568737, 1101153, 4810724, 2505407, 1906264, 2201333, 852904, 4357862, 3177610, 4964459, 1669181, 2352594, 120486961, 1268421, 1646970, 6886998, 121079291, 4864325, 1279693, 163801896, 1621736, 2329360, 737523, 1016832, 4022970, 4027045, 1403637, 3287565, 1015604, 3226526, 181540187, 24793812, 849389, 615272, 1273545, 3356062, 696291, 4852464, 2961648, 1968143, 109496655, 87683061, 682436, 63683565, 2193718, 2630902, 3139802, 3758355, 3706135, 14239555, 4081790, 94905053, 1410601, 2179357, 36645980, 23755802, 65679315, 2318257, 1545486, 572622, 4565484, 4083873, 3248052, 2544891, 1113795, 54868684, 7185815, 1905541, 887509, 1443596, 151410597, 52257607, 1242462, 3384958, 1582667, 2428686, 5974828], "NumGC":14158} Allocated: 2543335800, active: 2545192960, metadata: 21678544 (n_thp 0), resident: 2599702528, mapped: 2615087104, retained: 3238064128 2021-09-13 16:29:11 [ss-workload] instance inse% dele% read% 2021-09-13 16:29:11 [ss-workload] 0 14% 14% 70% 2021-09-13 16:29:11 [ss-workload] 1 14% 14% 70% Mon Sep 13 16:29:14 PDT 2021 RSS = 2956668 2021-09-13 16:29:11 [ss-workload] 2 14% 14% 70% 2021-09-13 16:29:11 [ss-workload] 3 14% 14% 70% 2021-09-13 16:29:11 [ss-workload] 4 3% 3% 92% 2021-09-13 16:29:11 [ss-workload] 5 3% 3% 92% 2021-09-13 16:29:11 [ss-workload] 6 3% 3% 92% 2021-09-13 16:29:11 [ss-workload] 7 18% 18% 62% 2021-09-13 16:29:11 [ss-workload] 8 18% 18% 62% 2021-09-13 16:29:11 [ss-workload] 9 18% 18% 62% 2021-09-13 16:29:11 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:29:11 [ss-diag] 0 56313 54973 0 0 282 1 18484536 4 88050868 196812 1.000000 13.738566 1.000000 0:0:0 2021-09-13 16:29:11 [ss-diag] 1 55187 53795 0 0 266 1 17435768 4 91461252 196764 1.000000 10.175839 1.000000 0:0:0 2021-09-13 16:29:11 [ss-diag] 2 55459 54137 0 0 241 1 15797068 4 86870636 196780 1.000000 0.952884 1.000000 0:0:0 2021-09-13 16:29:21 [gsi-test] [] QuotaTuner: 2311053234 -> 2079947910 2021-09-13 16:29:11 [ss-diag] 3 53733 52263 0 0 268 1 17566864 4 96576444 196764 1.000000 0.952884 1.000000 0:0:0 2021-09-13 16:29:11 [ss-diag] 4 2642394 2641194 0 0 4607 781 301979636 208 92248884 13577836 1.000000 2.480324 1.000000 0:0:0 2021-09-13 16:29:11 [ss-diag] 5 2678914 2677726 0 0 4881 837 319939788 208 91461124 13578028 1.000000 0.952884 1.000000 0:0:0 2021-09-13 16:29:11 [ss-diag] 6 2699511 2698307 0 0 4289 702 281135372 208 92510340 13577692 1.000000 0.952884 1.000000 0:0:0 2021-09-13 16:29:11 [ss-diag] 7 267172 265896 0 0 390 63 25563720 208 97242084 13577644 1.000000 3.682735 1.000000 0:0:0 2021-09-13 16:29:11 [ss-diag] 8 89509 87989 0 0 257 1 16845836 4 99856356 196780 1.000000 0.952884 1.000000 0:0:0 2021-09-13T16:29:29.057-07:00 [Info] memstats {"Alloc":425617344, "TotalAlloc":2505801744264, "Sys":976251128, "Lookups":0, "Mallocs":136413317,"Frees":136381600, "HeapAlloc":425617344, "HeapSys":934117376, "HeapIdle":506044416, "HeapInuse":428072960,"HeapReleased":430178304, "HeapObjects":31717,"GCSys":31092736, "LastGC":1631575768952942260,"PauseTotalNs":96375157270, "PauseNs":[1729656, 2238662, 1508499, 845496, 740795, 208172252, 3336554, 1913281, 1717860, 1547924, 114120205, 1283549, 167703726, 3148076, 1027504, 180952605, 13129228, 998764, 1825988, 6305996, 5695994, 2332079, 2758110, 1151686, 1414058, 1402936, 2675604, 886594, 674830, 5159585, 854875, 5095337, 3241400, 1599430, 949939, 47018493, 1089308, 1566347, 1139600, 1596002, 2172641, 3624475, 1767035, 5909081, 1792733, 1486897, 777246, 834577, 1208965, 789050, 1709977, 691880, 5575148, 1282113, 1906624, 68348368, 33923229, 1277382, 2906321, 3690726, 3032212, 1089613, 660533, 1736321, 3220228, 8093421, 3922800, 1818936, 1786393, 1982081, 2281135, 3065864, 2797944, 2477279, 1109239, 932446, 3383400, 118584350, 2635223, 1445634, 369471, 1630788, 4135397, 2311465, 1602641, 1282127, 32638657, 1498374, 1609365, 2784136, 1546452, 3649249, 1553464, 3546117, 1734428, 741587, 1826123, 1419436, 856865, 113169639, 4430212, 1752596, 4979190, 105600208, 1892616, 2420259, 145244377, 4057177, 1877977, 1925524, 938941, 1120417, 1246174, 3235892, 5061921, 3132305, 2122802, 3058865, 956679, 3321846, 1825431, 941258, 2227656, 2531673, 1757205, 4139695, 1570582, 136914850, 1274745, 2876664, 193233069, 726156, 1540200, 871168, 2206388, 1660717, 1615059, 2610745, 1677101, 1222696, 1465640, 306632, 1696525, 697983, 1673170, 6391878, 4041679, 2059261, 1071802, 2292797, 950727, 4687238, 630130, 1280096, 3106092, 1465695, 1779534, 135354176, 1674258, 1653403, 1716025, 162043052, 1142745, 2504775, 6752647, 2097767, 1872903, 2693779, 5906724, 4013383, 3587502, 1148961, 2812035, 1017587, 1081799, 782192, 1604151, 3435165, 1207019, 2047960, 1162890], "NumGC":14339} Allocated: 2474148504, active: 2475986944, metadata: 21678544 (n_thp 0), resident: 2596716544, mapped: 2612101120, retained: 3241050112 2021-09-13 16:29:11 [ss-diag] 9 259202 257912 0 0 312 52 20450976 208 98152060 13577804 1.000000 0.952884 1.000000 0:0:0 2021-09-13T16:29:59.251-07:00 [Info] memstats {"Alloc":418564408, "TotalAlloc":2549859510928, "Sys":976251128, "Lookups":0, "Mallocs":139870691,"Frees":139842888, "HeapAlloc":418564408, "HeapSys":934445056, "HeapIdle":513196032, "HeapInuse":421249024,"HeapReleased":395542528, "HeapObjects":27803,"GCSys":30896128, "LastGC":1631575799130311978,"PauseTotalNs":97825876626, "PauseNs":[3705870, 2380917, 1116928, 1110119, 1943548, 3245700, 5125972, 2951860, 842738, 4377771, 2262044, 2481815, 1512716, 2068390, 1256239, 2017717, 1949866, 986657, 2864625, 1149801, 1051659, 4494340, 913938, 1041794, 4373231, 1383065, 1767564, 4709515, 1025261, 7970971, 1128148, 925060, 5031328, 637537, 2426355, 2288027, 887539, 4720213, 86041697, 393954, 2861402, 1511905, 1458256, 4284261, 1572878, 1689138, 645506, 1766557, 1991801, 2085211, 1144926, 4692060, 4499853, 2272777, 1619719, 3660796, 2206297, 10121218, 15301821, 1814750, 1463609, 3874655, 2046008, 624947, 1656084, 2935705, 865643, 4857455, 1661940, 3452373, 3149527, 1468321, 3387491, 1997396, 333124, 2340223, 3330547, 1266895, 1102214, 1240713, 880415, 1896962, 4874457, 2401126, 1020071, 1065540, 157396441, 1915534, 3717993, 784083, 846951, 1316006, 1439021, 3583618, 1395941, 1358650, 1693616, 2286253, 3753525, 4127131, 4234370, 1590940, 1365965, 5371803, 2454694, 879560, 2598148, 1149789, 945327, 771756, 71534099, 85314391, 1150645, 2358474, 2098980, 2004166, 1076347, 81294053, 972459, 873624, 47947966, 1628131, 530730, 3574113, 686351, 1386391, 1952352, 1214658, 4538926, 917314, 1030688, 1828388, 3023058, 196659101, 677271, 1219898, 982508, 2027295, 1188829, 1222454, 145444485, 3272126, 1946172, 1620167, 1090582, 2769438, 1652491, 1294046, 874037, 2929739, 1512089, 1379335, 2163750, 3690477, 764114, 785122, 3589592, 2486766, 950036, 2190618, 658706, 1761990, 1369395, 1497130, 5851090, 1759005, 5087358, 881245, 637023, 2907842, 978217, 92214785, 3657116, 1149237, 1554232, 1132417, 2768917, 1801064, 427671, 98265004, 1382382, 1277606, 883467, 6225500, 1337938], "NumGC":14524} Allocated: 2084210784, active: 2086023168, metadata: 21678544 (n_thp 0), resident: 2218405888, mapped: 2235363328, retained: 3617787904 Mon Sep 13 16:30:14 PDT 2021 RSS = 2759508 2021-09-13 16:30:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 229, "num_active_sctxs": 44, "num_free_sctxs": 188, "num_trim_sctxs": 93, "num_wctxs": 191, "num_free_wctxs": 170, "num_active_wctxs": 20, "num_trim_wctxs": 922, "num_readers": 71, "num_writers": 86, "cpu": 9.05524, "total_core": 16, "threads": 54, "assigned_quota": 2079947910, "current_quota": 2079947910, "memory_resident": 0, "memory_jemalloc": 2138136576, "memory_jemalloc_allocated": 2041911184, "memory_in_use": 2194284544, "memory_stats_size_total": 2731904869, "memory_stats_size_global": 986428008, "memory_stats_size_page": 917641440, "memory_stats_size_index": 68872904, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 140344, "reclaim_pending": 19866524, "buf_memused": 1725524001, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 22, "num_active_smr": 10, "num_pending_smr": 10, "num_pages": 1060, "items_count": 2080, "total_records": 12946, "total_ops": 759726, "compacts": 1466666, "purges": 0, "lss_data_size": 188654118, "lss_used_space": 689236973, "lss_fragmentation": 72, "lss_num_reads": 172252, "lss_read_bs": 43347072772, "lss_blk_read_bs": 44198408192, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 44002729984, "bytes_incoming": 24290122360, "write_amp_avg": 0.86, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 232 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 12946, "resident_ratio": 100.00000, "cleaner_num_reads": 172252, "cleaner_read_bs": 43347072772, "cleaner_blk_read_bs": 44198408192, "recovery_num_lssctx": 5, "recovery_num_sctxs": 28, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 8, "recovery_num_trim_sctxs": 8, "recovery_buf_memused": 3145728, "recovery_data_size": 48142564, "recovery_used_space": 196721964, "recovery_lss_blk_read_bs": 18882883584, "recovery_lss_read_bs": 18193908345, "recovery_lss__num_reads": 142099, "recovery_bytes_written": 18421755904, "recovery_lss_frag": 75, "recovery_cleaner_num_reads": 142099, "recovery_cleaner_read_bs": 18193908345, "recovery_cleaner_blk_read_bs": 18882883584, "mvcc_purge_ratio": 6.22404 }2021-09-13T16:30:29.548-07:00 [Info] memstats {"Alloc":486544944, "TotalAlloc":2586424644816, "Sys":976251128, "Lookups":0, "Mallocs":142767451,"Frees":142733629, "HeapAlloc":486544944, "HeapSys":933986304, "HeapIdle":445136896, "HeapInuse":488849408,"HeapReleased":415416320, "HeapObjects":33822,"GCSys":30961664, "LastGC":1631575829468276802,"PauseTotalNs":101196500639, "PauseNs":[3663637, 2804918, 1630486, 1776009, 907863, 876031, 1144306, 5908314, 2437535, 3470590, 747822, 262873170, 6882753, 3667028, 795883, 935011, 1513100, 1323791, 1326229, 1179762, 1127806, 1339490, 1338741, 1082846, 894446, 5381735, 125986940, 1710968, 1073432, 4229622, 60782899, 839802, 4485900, 1196771, 5077246, 4959983, 3047229, 1017144, 1906383, 2958881, 1028170, 870190, 961087, 1527396, 1527227, 4730879, 1430251, 4016872, 2580534, 1628486, 1174355, 1565393, 654628, 3047861, 1151140, 74576602, 171349028, 1510202, 776579, 1121651, 1277515, 1567038, 2591459, 1683294, 4752058, 112650964, 1361077, 1338812, 1604542, 3059533, 1312012, 144762909, 1216822, 1121915, 2481357, 1538902, 93188384, 1761390, 1465582, 426719, 3055643, 5174955, 130139298, 3658877, 68101322, 42187881, 4052724, 4973784, 4081685, 1861415, 3853456, 1108131, 982878, 1723751, 1596972, 1927667, 4292156, 88462630, 4604945, 4003257, 944015, 3557681, 740494, 173836354, 1366185, 1788836, 1137445, 1257427, 193961153, 150281997, 92364184, 7228315, 3247557, 69925908, 1624817, 5079965, 1627657, 133167832, 1536234, 911171, 2816004, 2090936, 85132721, 1461772, 1227934, 3933976, 2804739, 103712403, 902205, 82127783, 653861, 1484946, 3401529, 50757872, 22980361, 1240092, 116330552, 1679151, 3123428, 264419207, 5854449, 1689231, 59421447, 1121131, 1249769, 115646180, 1766757, 2273756, 1919532, 3061943, 1740905, 2475470], "NumGC":14676} Allocated: 2091540480, active: 2093338624, metadata: 21678544 (n_thp 0), resident: 2371244032, mapped: 2386628608, retained: 3466522624 2021-09-13 16:30:29 [ss-workload] instance inse% dele% read% 2021-09-13 16:30:29 [ss-workload] 0 14% 14% 70% 2021-09-13 16:30:29 [ss-workload] 1 14% 14% 70% 2021-09-13 16:30:29 [ss-workload] 2 14% 14% 70% 2021-09-13 16:30:33 [gsi-test] [9] CreateRecoveryPoint() - DO!!! 2021-09-13 16:30:29 [ss-workload] 3 14% 14% 70% 2021-09-13 16:30:29 [ss-workload] 4 3% 3% 92% 2021-09-13 16:30:29 [ss-workload] 5 3% 3% 92% 2021-09-13 16:30:29 [ss-workload] 6 3% 3% 92% 2021-09-13 16:30:29 [ss-workload] 7 18% 18% 62% 2021-09-13 16:30:29 [ss-workload] 8 18% 18% 62% 2021-09-13 16:30:29 [ss-workload] 9 18% 18% 62% 2021-09-13 16:30:29 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:30:29 [ss-diag] 0 58303 56841 0 0 275 1 18025700 4 96053052 196812 1.000000 3.975215 1.000000 0:0:0 2021-09-13 16:30:37 [gsi-test] [6] CreateRecoveryPoint() - DO!!! 2021-09-13 16:30:29 [ss-diag] 1 57146 55668 0 0 268 1 17566864 4 97102620 196764 1.000000 2.631106 1.000000 0:0:0 2021-09-13 16:30:29 [ss-diag] 2 57815 56478 0 0 245 1 16059260 4 87854040 196780 1.000000 0.874117 1.000000 0:0:0 2021-09-13 16:30:29 [ss-diag] 3 55825 54153 0 0 263 1 17239124 4 109825636 196764 1.000000 0.874117 1.000000 0:0:0 2021-09-13 16:30:29 [ss-diag] 4 2924507 2923301 0 0 4077 684 267239196 208 92642220 13577836 1.000000 2.440314 1.000000 0:0:0 2021-09-13 16:30:29 [ss-diag] 5 2964622 2963406 0 0 3862 663 253146376 208 93296692 13578028 1.000000 0.874117 1.000000 0:0:0 2021-09-13 16:30:40 [gsi-test] [8] CreateRecoveryPoint() - DO!!! 2021-09-13 16:30:29 [ss-diag] 6 2988147 2986789 0 0 3338 591 218799224 208 102605964 13577692 1.000000 0.874117 1.000000 0:0:0 2021-09-13 16:30:41 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 16:30:41 [gsi-test] [5] CreateRecoveryPoint() - DO!!! 2021-09-13 16:30:41 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 16:30:29 [ss-diag] 7 289955 288653 0 0 307 50 20123236 208 98946220 13577644 1.000000 3.942770 1.000000 0:0:0 2021-09-13 16:30:29 [ss-diag] 8 93526 91750 0 0 262 1 17173576 4 116645924 196780 1.000000 0.874117 1.000000 0:0:0 2021-09-13 16:30:29 [ss-diag] 9 282914 281576 0 0 352 53 23072896 208 101302140 13577804 1.000000 1.118475 1.000000 0:0:0 2021-09-13 16:30:48 [gsi-test] [2] CreateRecoveryPoint() - DO!!! 2021-09-13 16:30:49 [gsi-test] [3] CreateRecoveryPoint() - DO!!! 2021-09-13 16:30:49 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 16:30:59 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: starting... frag 36, data: 35575651, used: 56094921 log:(1604300599 - 1660395520) head:1591060748 2021-09-13 16:30:59 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: completed... frag 55, data: 59806187, used: 133512999, disk: 154320124, relocated: 3543, retries: 2374, skipped: 160, cleaned:1932943778, log:(3537244377 - 3670757376), disk:(3516437252-3670757376), activeFragRatio:30, activeMaxFragRatio:80, run:32 duration:102986 ms elapsed: 301261 ms 2021-09-13T16:31:00.165-07:00 [Info] memstats {"Alloc":470829576, "TotalAlloc":2621656888328, "Sys":976251128, "Lookups":0, "Mallocs":145725107,"Frees":145688180, "HeapAlloc":470829576, "HeapSys":934084608, "HeapIdle":460767232, "HeapInuse":473317376,"HeapReleased":365461504, "HeapObjects":36927,"GCSys":30994432, "LastGC":1631575860164325828,"PauseTotalNs":103868518340, "PauseNs":[1228799, 1428553, 3716500, 1607171, 114419240, 1448801, 1531563, 1533841, 1546202, 1723969, 532221, 1559488, 212640269, 1299884, 1911773, 1250133, 2690067, 3270733, 2322963, 1667580, 4708644, 5487461, 4089518, 1959300, 1407490, 2048235, 1375010, 1528706, 2325781, 2491749, 2711692, 581605, 160399284, 1849843, 1205129, 4879007, 86125404, 1287165, 1685470, 1523684, 828002, 5073436, 1329537, 1821052, 1060346, 2420237, 7421075, 4824643, 3801273, 4205130, 847310, 1291148, 1399013, 2627178, 1161121, 4629575, 1652412, 1575119, 1308855, 3251548, 1625913, 1368773, 3606713, 1333764, 3608077, 5213911, 3256819, 8856038, 9100501, 1279374, 5996878, 2358714, 5515350, 12036743, 14627646, 10268822, 2739857, 14692602, 13351133, 14152153, 18216256, 13828176, 12352711, 1999009, 7516737, 17940192, 80711881, 1776920, 21779870, 6278173, 125417751, 4447232, 19940844, 12978520, 37291765, 12160217, 15392510, 1763894, 6267634, 5167301, 3268369, 40382933, 3259528, 22063480, 13001157, 11440226, 28183579, 18917241, 28050380, 39865409, 33873772, 31402453, 41949435, 53093114, 11134756, 153658486, 23039631, 28678083, 42968684, 31061558, 48116936, 17301453, 33182427, 109222522, 27015425, 13213836, 168874959, 14388883, 18447734, 18103898, 24556485, 2635630, 315100, 6524875, 807865, 43272045, 2243445, 6359954, 793139, 1153603, 4714511, 95673747, 981036, 1295512, 93785096], "NumGC":14821} Allocated: 2201032592, active: 2202750976, metadata: 21714000 (n_thp 0), resident: 2417913856, mapped: 2433290240, retained: 3587633152 2021-09-13 16:31:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: starting... frag 37, data: 38506587, used: 61381943 log:(3805507938 - 3867140096) head:3799928959 2021-09-13 16:31:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: completed... frag 30, data: 42970817, used: 61450312, disk: 71633745, relocated: 4076167, retries: 5516, skipped: 71460, cleaned:1488466310, log:(5293974248 - 5356388352), disk:(5284754607-5356388352), activeFragRatio:30, activeMaxFragRatio:80, run:250 duration:100668 ms elapsed: 300075 ms 2021-09-13 16:31:03 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 95, data: 2106368, used: 44834658 log:(31433048204 - 31477882880) head:31431995462 2021-09-13 16:31:03 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 94, data: 3155424, used: 56090396, disk: 56618530, relocated: 49018793, retries: 285523, skipped: 12588386, cleaned:9684013394, log:(41117061598 - 41173680128), disk:(41117061598-41173680128), activeFragRatio:30, activeMaxFragRatio:80, run:225 duration:287054 ms elapsed: 300265 ms 2021-09-13 16:31:03 [gsi-test] [1] CreateRecoveryPoint() - DO!!! Mon Sep 13 16:31:14 PDT 2021 RSS = 2881260 2021-09-13 16:31:15 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 52, data: 15350700, used: 32390434 log:(27229106386 - 27262021632) head:27228053644 2021-09-13 16:31:15 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 40, data: 18366736, used: 30813718, disk: 31997672, relocated: 63290887, retries: 27204, skipped: 4724270, cleaned:8918372562, log:(36147478948 - 36178423808), disk:(36146426136-36178423808), activeFragRatio:30, activeMaxFragRatio:80, run:281 duration:265989 ms elapsed: 300714 ms 2021-09-13 16:31:19 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: starting... frag 89, data: 6149845, used: 58331019 log:(14708736117 - 14767067136) head:14687774085 2021-09-13 16:31:19 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: completed... frag 91, data: 10610603, used: 124285463, disk: 127177661, relocated: 4263949, retries: 97617, skipped: 24357, cleaned:4152936485, log:(18861672602 - 18986278912), disk:(18859101251-18986278912), activeFragRatio:30, activeMaxFragRatio:80, run:76 duration:289391 ms elapsed: 302410 ms 2021-09-13 16:31:22 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: starting... frag 81, data: 104493659, used: 580076044 log:(27133366426 - 27714084864) head:27131076048 2021-09-13 16:31:22 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: completed... frag 83, data: 125956877, used: 777619986, disk: 799517382, relocated: 303213, retries: 19927, skipped: 8353, cleaned:5065790465, log:(32199156891 - 32977776640), disk:(32178259258-32977776640), activeFragRatio:30, activeMaxFragRatio:80, run:10 duration:325794 ms elapsed: 335558 ms 2021-09-13T16:31:30.703-07:00 [Info] memstats {"Alloc":415076288, "TotalAlloc":2661096689352, "Sys":976251128, "Lookups":0, "Mallocs":149170105,"Frees":149136988, "HeapAlloc":415076288, "HeapSys":934379520, "HeapIdle":516939776, "HeapInuse":417439744,"HeapReleased":419422208, "HeapObjects":33117,"GCSys":31059968, "LastGC":1631575890465063611,"PauseTotalNs":105394166223, "PauseNs":[1313742, 7539940, 87406419, 2464017, 1710581, 47873282, 2355414, 1839989, 3511280, 3291146, 795458, 1499304, 946545, 4573063, 945014, 1669297, 3076012, 2986734, 6045514, 5681968, 108577993, 39436412, 137698572, 26172722, 7032235, 1341716, 1337241, 5463870, 37127110, 16308210, 37505033, 1269335, 112671757, 30360809, 43742096, 114124842, 3752512, 27223238, 6309168, 583548, 14354248, 611940, 25452223, 4257920, 2106854, 1078792, 1613842, 99822803, 1143959, 3766078, 14946345, 695058, 1896445, 1371445, 1275199, 3752898, 3262051, 1073119, 1762197, 2906252, 102888423, 5814494, 2310569, 463038, 1193135, 1635874, 5407441, 4233051, 3125850, 908006, 1046149, 1222341, 44183485, 932212, 1278691, 1605933, 4312138, 3878478, 2483393, 1935903, 1396268, 676392, 2843684, 4765712, 8061527, 4158967, 1837712, 2072745, 1622870, 5752259, 872946, 929951, 1397265, 6468209, 1354268, 2597275, 1462881, 1499365, 976691, 3090271, 1664502, 5949668, 3326640, 4747704, 989789, 1240165, 2133564, 4756094, 590414, 1184407, 1368388, 894635, 3689268, 1616040, 1702325, 4344405, 755902, 1550865, 1846782, 1923404, 4850464, 1749756, 1321643, 1393288, 709649, 4728971, 5262479, 3962937, 1188015, 1376808, 1734568, 1796729, 1691659, 4047524, 1044411, 1520795, 1287283, 5362008, 1496096, 1739358, 2257516, 3519800, 1450115, 1012388, 3774489, 3770745, 3034134, 1287648, 1790025, 3874503, 791972, 5016487, 1329627, 1512568, 1132671, 979669, 791705, 2408905, 1128857, 1774948, 2576607, 3092816, 2834253, 1320040, 1237325], "NumGC":14986} Allocated: 2329039576, active: 2330812416, metadata: 21714000 (n_thp 0), resident: 2506178560, mapped: 2521554944, retained: 3667140608 2021-09-13 16:31:36 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: starting... frag 31, data: 46600738, used: 67803295 log:(1533884257 - 1601687552) head:1507092939 2021-09-13 16:31:36 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: completed... frag 64, data: 37582249, used: 104726528, disk: 129774512, relocated: 2903, retries: 1917, skipped: 331, cleaned:1719182075, log:(3253066332 - 3357794304), disk:(3228019792-3357794304), activeFragRatio:30, activeMaxFragRatio:80, run:27 duration:89857 ms elapsed: 310576 ms 2021-09-13 16:31:42 [gsi-test] [] { "memory_quota": 2079947910, "count": 24606, "compacts": 124, "purges": 0, "splits": 3, "merges": 0, "inserts": 24606, "deletes": 0, "compact_conflicts": 241, "split_conflicts": 10, "merge_conflicts": 0, "insert_conflicts": 3134, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 112181428, "memory_size_index": 196812, "allocated": 3964190192, "freed": 3852008764, "reclaimed": 3852008764, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 196812, "freed_index": 0, "reclaimed_index": 0, "num_pages": 4, "items_count": 208, "total_records": 1708, "num_rec_allocs": 60403, "num_rec_frees": 58695, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1612874088, "lss_data_size": 48096493, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 63284859, "est_recovery_size": 9898788, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 73430, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 48824, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 8.21154, "currSn": 1767, "gcSn": 1763, "gcSnIntervals": "[0 581 1144 1716 1764]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 23, "num_trim_wctxs": 83, "num_readers": 8, "num_writers": 7, "page_bytes": 21958580, "page_cnt": 1, "page_itemcnt": 335, "avg_item_size": 65548, "avg_page_size": 21958580, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 100837600, "page_bytes_marshalled":3864277936, "page_bytes_compressed":2103290770, "compression_ratio": 1.83725, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 18768, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 125539817, "lss_data_size": 48498181, "lss_used_space": 64230252, "lss_disk_size": 94793728, "lss_fragmentation": 24, "lss_num_reads": 621, "lss_read_bs": 3885635896, "lss_blk_read_bs": 3897729024, "bytes_written": 3919998976, "bytes_incoming": 1612874088, "write_amp": 9.76, "write_amp_avg": 2.41, "lss_gc_num_reads": 621, "lss_gc_reads_bs": 3885635896, "lss_blk_gc_reads_bs": 3897729024, "lss_gc_status": "frag 0, data: 48096493, used: 47678538, disk: 86107578, relocated: 650, retries: 469, skipped: 20, cleaned:305814237, log:(3843058614 - 3890737152), disk:(3804629574-3890737152), activeFragRatio:30, activeMaxFragRatio:80, run:4 duration:11433 ms elapsed: 34643 ms", "lss_head_offset": 3843058614, "lss_tail_offset": 3890737152, "num_sctxs": 31, "num_free_sctxs": 23, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:31:43 [gsi-test] [] { "memory_quota": 2079947910, "count": 24164, "compacts": 122, "purges": 0, "splits": 3, "merges": 0, "inserts": 24164, "deletes": 0, "compact_conflicts": 178, "split_conflicts": 8, "merge_conflicts": 0, "insert_conflicts": 2788, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 100248924, "memory_size_index": 196764, "allocated": 3875084636, "freed": 3774835712, "reclaimed": 3774835712, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 196764, "freed_index": 0, "reclaimed_index": 0, "num_pages": 4, "items_count": 208, "total_records": 1526, "num_rec_allocs": 59045, "num_rec_frees": 57519, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1583901872, "lss_data_size": 53310231, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 65012476, "est_recovery_size": 10304578, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 72201, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 48037, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.33654, "currSn": 1767, "gcSn": 1764, "gcSnIntervals": "[0 579 1155 1728 1765]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 21, "num_trim_wctxs": 89, "num_readers": 7, "num_writers": 4, "page_bytes": 20254332, "page_cnt": 1, "page_itemcnt": 309, "avg_item_size": 65548, "avg_page_size": 20254332, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 92838600, "page_bytes_marshalled":3706308260, "page_bytes_compressed":2017316803, "compression_ratio": 1.83725, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 16936, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 131111317, "lss_data_size": 53711919, "lss_used_space": 65518394, "lss_disk_size": 130965504, "lss_fragmentation": 18, "lss_num_reads": 552, "lss_read_bs": 3630566027, "lss_blk_read_bs": 3639554048, "bytes_written": 3486408704, "bytes_incoming": 1583901872, "write_amp": 4.83, "write_amp_avg": 2.18, "lss_gc_num_reads": 552, "lss_gc_reads_bs": 3630566027, "lss_blk_gc_reads_bs": 3639554048, "lss_gc_status": "frag 0, data: 53310231, used: 49286354, disk: 102052263, relocated: 303, retries: 192, skipped: 32, cleaned:157493458, log:(3410559790 - 3459846144), disk:(3357793881-3459846144), activeFragRatio:30, activeMaxFragRatio:80, run:2 duration:6116 ms elapsed: 6116 ms", "lss_head_offset": 3410559790, "lss_tail_offset": 3459846144, "num_sctxs": 29, "num_free_sctxs": 21, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:31:44 [gsi-test] [] { "memory_quota": 2079947910, "count": 24098, "compacts": 122, "purges": 0, "splits": 3, "merges": 0, "inserts": 24098, "deletes": 0, "compact_conflicts": 221, "split_conflicts": 6, "merge_conflicts": 0, "insert_conflicts": 2807, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 8, "memory_size": 105883028, "memory_size_index": 196780, "allocated": 3934605180, "freed": 3828722152, "reclaimed": 3828722152, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 196780, "freed_index": 0, "reclaimed_index": 0, "num_pages": 4, "items_count": 208, "total_records": 1612, "num_rec_allocs": 59951, "num_rec_frees": 58339, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1579575704, "lss_data_size": 25803722, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 215031016, "est_recovery_size": 664294, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 72152, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 48054, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.75000, "currSn": 1680, "gcSn": 1677, "gcSnIntervals": "[0 466 1047 1626 1678]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 25, "num_trim_wctxs": 85, "num_readers": 9, "num_writers": 7, "page_bytes": 20254332, "page_cnt": 1, "page_itemcnt": 309, "avg_item_size": 65548, "avg_page_size": 20254332, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 101493384, "page_bytes_marshalled":3223021034, "page_bytes_compressed":1754283664, "compression_ratio": 1.83723, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12552, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 320820576, "lss_data_size": 112067115, "lss_used_space": 1001667527, "lss_disk_size": 1087651840, "lss_fragmentation": 88, "lss_num_reads": 79899, "lss_read_bs": 37657695055, "lss_blk_read_bs": 38744920064, "bytes_written": 38467289088, "bytes_incoming": 33324734296, "write_amp": 1.12, "write_amp_avg": 1.00, "lss_gc_num_reads": 79899, "lss_gc_reads_bs": 37657695055, "lss_blk_gc_reads_bs": 38744920064, "lss_gc_status": "frag 83, data: 125956877, used: 777619986 log:(32199156891 - 32977776640) head:32178259258", "lss_head_offset": 32542788609, "lss_tail_offset": 33352400896, "num_sctxs": 81, "num_free_sctxs": 70, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:31:45 [gsi-test] [] { "memory_quota": 2079947910, "count": 23536, "compacts": 118, "purges": 0, "splits": 3, "merges": 0, "inserts": 23536, "deletes": 0, "compact_conflicts": 234, "split_conflicts": 5, "merge_conflicts": 0, "insert_conflicts": 3078, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 112315356, "memory_size_index": 196764, "allocated": 3789773776, "freed": 3677458420, "reclaimed": 3677458420, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 196764, "freed_index": 0, "reclaimed_index": 0, "num_pages": 4, "items_count": 208, "total_records": 1710, "num_rec_allocs": 57743, "num_rec_frees": 56033, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1542737728, "lss_data_size": 26661748, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 222181233, "est_recovery_size": 664150, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 70482, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 46946, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 8.22115, "currSn": 1670, "gcSn": 1666, "gcSnIntervals": "[0 468 1050 1615 1667]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 21, "num_trim_wctxs": 82, "num_readers": 9, "num_writers": 6, "page_bytes": 19336660, "page_cnt": 1, "page_itemcnt": 295, "avg_item_size": 65548, "avg_page_size": 19336660, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 90343312, "page_bytes_marshalled":3255075294, "page_bytes_compressed":1771719868, "compression_ratio": 1.83724, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 22336, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 320820576, "lss_data_size": 116492521, "lss_used_space": 1004367734, "lss_disk_size": 1117487104, "lss_fragmentation": 88, "lss_num_reads": 79917, "lss_read_bs": 37673610755, "lss_blk_read_bs": 38761635840, "bytes_written": 38497124352, "bytes_incoming": 33339810336, "write_amp": 1.18, "write_amp_avg": 1.00, "lss_gc_num_reads": 79917, "lss_gc_reads_bs": 37673610755, "lss_blk_gc_reads_bs": 38761635840, "lss_gc_status": "frag 83, data: 125956877, used: 777619986 log:(32199156891 - 32977776640) head:32178259258", "lss_head_offset": 32568478194, "lss_tail_offset": 33381314560, "num_sctxs": 81, "num_free_sctxs": 69, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:31:45 [gsi-test] [] { "memory_quota": 2079947910, "count": 191294, "compacts": 508349, "purges": 0, "splits": 207, "merges": 0, "inserts": 191294, "deletes": 0, "compact_conflicts": 246, "split_conflicts": 28, "merge_conflicts": 0, "insert_conflicts": 2419, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 121224828, "memory_size_index": 13577836, "allocated": 243665294660, "freed": 243544069832, "reclaimed": 243538956032, "reclaim_pending": 5113800, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577836, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1642, "num_rec_allocs": 3210422, "num_rec_frees": 3208780, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 12538939112, "lss_data_size": 9293335, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 116166687, "est_recovery_size": 52858604, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 3156266, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 2964972, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.89423, "currSn": 31687, "gcSn": 31670, "gcSnIntervals": "[0 9651 20176 30592 31671]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 32, "num_trim_wctxs": 105, "num_readers": 10, "num_writers": 14, "page_bytes": 429011660, "page_cnt": 825, "page_itemcnt": 6545, "avg_item_size": 65548, "avg_page_size": 520014, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 132573504, "page_bytes_marshalled":30439220156, "page_bytes_compressed":16586217668, "compression_ratio": 1.83521, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 13440, "num_total_smr": 10, "num_active_smr": 2, "num_pending_smr": 7, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 117643300, "lss_data_size": 12711023, "lss_used_space": 174306930, "lss_disk_size": 231936000, "lss_fragmentation": 92, "lss_num_reads": 396762, "lss_read_bs": 61694016929, "lss_blk_read_bs": 62506438656, "bytes_written": 62039199744, "bytes_incoming": 12538939112, "write_amp": 2.66, "write_amp_avg": 1.55, "lss_gc_num_reads": 396762, "lss_gc_reads_bs": 61694016929, "lss_blk_gc_reads_bs": 62506438656, "lss_gc_status": "frag 93, data: 6896645, used: 108646851 log:(19209956793 - 19319246848) head:19197391464", "lss_head_offset": 19262739558, "lss_tail_offset": 19385028608, "num_sctxs": 43, "num_free_sctxs": 35, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:31:45 [gsi-test] [] { "memory_quota": 2079947910, "count": 190088, "compacts": 509393, "purges": 0, "splits": 207, "merges": 0, "inserts": 190088, "deletes": 0, "compact_conflicts": 280, "split_conflicts": 21, "merge_conflicts": 0, "insert_conflicts": 2197, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 15, "memory_size": 118994772, "memory_size_index": 13578028, "allocated": 245887236436, "freed": 245768241664, "reclaimed": 245767586056, "reclaim_pending": 655608, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13578028, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1608, "num_rec_allocs": 3243224, "num_rec_frees": 3241616, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 12459888224, "lss_data_size": 3435627, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 28630225, "est_recovery_size": 76799192, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 3193105, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 3003017, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.73077, "currSn": 31362, "gcSn": 31345, "gcSnIntervals": "[0 7887 19143 30229 31346]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 32, "num_trim_wctxs": 108, "num_readers": 12, "num_writers": 14, "page_bytes": 240036776, "page_cnt": 481, "page_itemcnt": 3662, "avg_item_size": 65548, "avg_page_size": 499036, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 132834640, "page_bytes_marshalled":6906484492, "page_bytes_compressed":3763045865, "compression_ratio": 1.83534, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12288, "num_total_smr": 2, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 320820576, "lss_data_size": 118896951, "lss_used_space": 1022557244, "lss_disk_size": 1071853568, "lss_fragmentation": 88, "lss_num_reads": 79927, "lss_read_bs": 37676967815, "lss_blk_read_bs": 38765637632, "bytes_written": 38518599680, "bytes_incoming": 33346627328, "write_amp": 1.18, "write_amp_avg": 1.00, "lss_gc_num_reads": 79927, "lss_gc_reads_bs": 37676967815, "lss_blk_gc_reads_bs": 38765637632, "lss_gc_status": "frag 83, data: 125956877, used: 777619986 log:(32199156891 - 32977776640) head:32178259258", "lss_head_offset": 32572478240, "lss_tail_offset": 33401868288, "num_sctxs": 81, "num_free_sctxs": 71, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:31:45 [gsi-test] [] { "memory_quota": 2079947910, "count": 190210, "compacts": 511823, "purges": 0, "splits": 207, "merges": 0, "inserts": 190210, "deletes": 0, "compact_conflicts": 298, "split_conflicts": 39, "merge_conflicts": 0, "insert_conflicts": 2406, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 13, "memory_size": 118863020, "memory_size_index": 13577692, "allocated": 247303165992, "freed": 247184302972, "reclaimed": 247182204988, "reclaim_pending": 2097984, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577692, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1606, "num_rec_allocs": 3262349, "num_rec_frees": 3260743, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 12467885080, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 68266, "est_recovery_size": 72798908, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 3205691, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 3015481, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.72115, "currSn": 31144, "gcSn": 31128, "gcSnIntervals": "[0 7609 18901 29933 31129]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 36, "num_trim_wctxs": 115, "num_readers": 13, "num_writers": 14, "page_bytes": 253867404, "page_cnt": 502, "page_itemcnt": 3873, "avg_item_size": 65548, "avg_page_size": 505711, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 133490504, "page_bytes_marshalled":6854568634, "page_bytes_compressed":3734747116, "compression_ratio": 1.83535, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 11648, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 320820576, "lss_data_size": 118896951, "lss_used_space": 1022557244, "lss_disk_size": 1071853568, "lss_fragmentation": 88, "lss_num_reads": 79927, "lss_read_bs": 37676967815, "lss_blk_read_bs": 38765637632, "bytes_written": 38518599680, "bytes_incoming": 33346627328, "write_amp": 1.18, "write_amp_avg": 1.00, "lss_gc_num_reads": 79927, "lss_gc_reads_bs": 37676967815, "lss_blk_gc_reads_bs": 38765637632, "lss_gc_status": "frag 83, data: 125956877, used: 777619986 log:(32199156891 - 32977776640) head:32178259258", "lss_head_offset": 32572478240, "lss_tail_offset": 33401868288, "num_sctxs": 81, "num_free_sctxs": 71, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:31:45 [ss-workload] instance inse% dele% read% 2021-09-13 16:31:45 [ss-workload] 0 14% 14% 70% 2021-09-13 16:31:47 [gsi-test] [] { "memory_quota": 2079947910, "count": 41568, "compacts": 42862, "purges": 0, "splits": 207, "merges": 0, "inserts": 41568, "deletes": 0, "compact_conflicts": 305, "split_conflicts": 30, "merge_conflicts": 0, "insert_conflicts": 4181, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 3, "memory_size": 123202004, "memory_size_index": 13577644, "allocated": 23395466256, "freed": 23272264252, "reclaimed": 23272264252, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577644, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1672, "num_rec_allocs": 313799, "num_rec_frees": 312127, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 2724699264, "lss_data_size": 49105500, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 73291791, "est_recovery_size": 32006980, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 113536, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 71968, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 8.03846, "currSn": 2050, "gcSn": 2047, "gcSnIntervals": "[0 665 1308 1978 2048]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 23, "num_trim_wctxs": 80, "num_readers": 3, "num_writers": 11, "page_bytes": 26743584, "page_cnt": 52, "page_itemcnt": 408, "avg_item_size": 65548, "avg_page_size": 514299, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 127602064, "page_bytes_marshalled":10045644626, "page_bytes_compressed":5473413846, "compression_ratio": 1.83535, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 23936, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 64454699, "lss_data_size": 67144424, "lss_used_space": 101205991, "lss_disk_size": 161452032, "lss_fragmentation": 33, "lss_num_reads": 309121, "lss_read_bs": 42524352032, "lss_blk_read_bs": 43792347136, "bytes_written": 42775580672, "bytes_incoming": 2724699264, "write_amp": 4.52, "write_amp_avg": 2.07, "lss_gc_num_reads": 309121, "lss_gc_reads_bs": 42524352032, "lss_blk_gc_reads_bs": 43792347136, "lss_gc_status": "frag 29, data: 49105500, used: 69998707, disk: 80429953, relocated: 745948, retries: 1006, skipped: 14328, cleaned:266646693, log:(5560620941 - 5630619648), disk:(5550189695-5630619648), activeFragRatio:30, activeMaxFragRatio:80, run:39 duration:16599 ms elapsed: 45183 ms", "lss_head_offset": 5560620941, "lss_tail_offset": 5630619648, "num_sctxs": 33, "num_free_sctxs": 25, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:31:48 [gsi-test] [] { "memory_quota": 2079947910, "count": 41738, "compacts": 209, "purges": 0, "splits": 3, "merges": 0, "inserts": 41738, "deletes": 0, "compact_conflicts": 466, "split_conflicts": 14, "merge_conflicts": 0, "insert_conflicts": 5954, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 128309036, "memory_size_index": 196780, "allocated": 6408341540, "freed": 6280032504, "reclaimed": 6280032504, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 196780, "freed_index": 0, "reclaimed_index": 0, "num_pages": 4, "items_count": 208, "total_records": 1954, "num_rec_allocs": 97622, "num_rec_frees": 95668, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 2735842424, "lss_data_size": 49389944, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 379922646, "est_recovery_size": 401842, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 109947, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 68209, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 9.39423, "currSn": 1909, "gcSn": 1902, "gcSnIntervals": "[0 508 1170 1832 1903]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 20, "num_trim_wctxs": 75, "num_readers": 4, "num_writers": 8, "page_bytes": 22482964, "page_cnt": 1, "page_itemcnt": 343, "avg_item_size": 65548, "avg_page_size": 22482964, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 121036976, "page_bytes_marshalled":3590814650, "page_bytes_compressed":1954486600, "compression_ratio": 1.83722, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 19856, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 320820576, "lss_data_size": 126323632, "lss_used_space": 1018791689, "lss_disk_size": 1120423936, "lss_fragmentation": 87, "lss_num_reads": 80073, "lss_read_bs": 37729247604, "lss_blk_read_bs": 38818168832, "bytes_written": 38567170048, "bytes_incoming": 33395263944, "write_amp": 1.18, "write_amp_avg": 1.00, "lss_gc_num_reads": 80073, "lss_gc_reads_bs": 37729247604, "lss_blk_gc_reads_bs": 38818168832, "lss_gc_status": "frag 83, data: 125956877, used: 777619986 log:(32199156891 - 32977776640) head:32178259258", "lss_head_offset": 32614078301, "lss_tail_offset": 33443852288, "num_sctxs": 81, "num_free_sctxs": 69, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:31:48 [gsi-test] [] Stop and verify... 2021-09-13 16:31:45 [ss-workload] 1 14% 14% 70% 2021-09-13 16:31:49 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper stopped 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:31:49 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : All daemons stopped 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper stopped 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:31:49 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : All daemons stopped 2021-09-13 16:31:45 [ss-workload] 2 14% 14% 70% 2021-09-13 16:31:49 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper stopped 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:31:49 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:31:50 [gsi-test] [] { "memory_quota": 2079947910, "count": 39166, "compacts": 41756, "purges": 0, "splits": 207, "merges": 0, "inserts": 39166, "deletes": 0, "compact_conflicts": 311, "split_conflicts": 32, "merge_conflicts": 0, "insert_conflicts": 3957, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 1, "memory_size": 123715572, "memory_size_index": 13577804, "allocated": 22748738004, "freed": 22625022432, "reclaimed": 22623973408, "reclaim_pending": 1049024, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577804, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1680, "num_rec_allocs": 305105, "num_rec_frees": 303425, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 2567252968, "lss_data_size": 7009766, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 58414716, "est_recovery_size": 31485340, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 108114, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 68948, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 8.07692, "currSn": 1934, "gcSn": 1931, "gcSnIntervals": "[0 504 1176 1854 1932]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 22, "num_trim_wctxs": 100, "num_readers": 3, "num_writers": 11, "page_bytes": 15862616, "page_cnt": 31, "page_itemcnt": 242, "avg_item_size": 65548, "avg_page_size": 511697, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 132970200, "page_bytes_marshalled":4777440554, "page_bytes_compressed":2603023521, "compression_ratio": 1.83534, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 13056, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 320820576, "lss_data_size": 118866472, "lss_used_space": 1039853235, "lss_disk_size": 1145307136, "lss_fragmentation": 88, "lss_num_reads": 80107, "lss_read_bs": 37746337446, "lss_blk_read_bs": 38836654080, "bytes_written": 38592053248, "bytes_incoming": 33426726984, "write_amp": 1.18, "write_amp_avg": 1.00, "lss_gc_num_reads": 80107, "lss_gc_reads_bs": 37746337446, "lss_blk_gc_reads_bs": 38836654080, "lss_gc_status": "frag 83, data: 125956877, used: 777619986 log:(32199156891 - 32977776640) head:32178259258", "lss_head_offset": 32614078301, "lss_tail_offset": 33468735488, "num_sctxs": 81, "num_free_sctxs": 71, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:31:50 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : All daemons stopped 2021-09-13 16:31:50 [gsi-test] [0] Stop and verify... 2021-09-13 16:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:31:45 [ss-workload] 3 14% 14% 70% 2021-09-13 16:31:45 [ss-workload] 4 3% 3% 92% 2021-09-13 16:31:45 [ss-workload] 5 3% 3% 92% 2021-09-13 16:31:45 [ss-workload] 6 3% 3% 92% 2021-09-13 16:31:45 [ss-workload] 7 18% 18% 62% 2021-09-13 16:31:52 [gsi-test] [0] Stop and verify... OK 2021-09-13 16:31:52 [gsi-test] [1] Stop and verify... 2021-09-13 16:31:45 [ss-workload] 8 18% 18% 62% 2021-09-13 16:31:54 [gsi-test] [1] Stop and verify... OK 2021-09-13 16:31:54 [gsi-test] [2] Stop and verify... 2021-09-13 16:31:45 [ss-workload] 9 18% 18% 62% 2021-09-13 16:31:45 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:31:45 [ss-diag] 0 60938 59186 0 0 335 1 21958580 4 115064708 196812 1.000000 7.067162 1.000000 0:0:0 2021-09-13 16:31:45 [ss-diag] 1 59537 57991 0 0 309 1 20254332 4 101558124 196764 1.000000 9.467104 1.000000 0:0:0 2021-09-13 16:31:55 [gsi-test] [2] Stop and verify... OK 2021-09-13 16:31:55 [gsi-test] [3] Stop and verify... 2021-09-13 16:31:57 [gsi-test] [3] Stop and verify... OK 2021-09-13 16:31:57 [gsi-test] [4] Stop and verify... 2021-09-13 16:31:57 [gsi-test] [4] Stop and verify... OK 2021-09-13 16:31:57 [gsi-test] [5] Stop and verify... 2021-09-13 16:31:57 [gsi-test] [5] Stop and verify... OK 2021-09-13 16:31:57 [gsi-test] [6] Stop and verify... 2021-09-13 16:31:57 [gsi-test] [6] Stop and verify... OK 2021-09-13 16:31:57 [gsi-test] [7] Stop and verify... 2021-09-13 16:31:58 [gsi-test] [7] Stop and verify... OK 2021-09-13 16:31:58 [gsi-test] [8] Stop and verify... 2021-09-13 16:31:45 [ss-diag] 2 60136 58339 0 0 309 1 20254332 4 118016872 196780 1.000000 1.179473 1.000000 0:0:0 2021-09-13 16:31:45 [ss-diag] 3 58646 56998 0 0 295 1 19336660 4 108242948 196764 1.000000 1.179473 1.000000 0:0:0 2021-09-13 16:31:45 [ss-diag] 4 3286273 3284647 0 0 5970 757 391321560 208 120174268 13577836 1.000000 2.660279 1.000000 0:0:0 2021-09-13 16:31:45 [ss-diag] 5 3320339 3318695 0 0 4101 532 268812348 208 121354148 13578028 1.000000 1.179473 1.000000 0:0:0 2021-09-13 16:31:45 [ss-diag] 6 3348519 3346909 0 0 4431 577 290443188 208 119125244 13577692 1.000000 1.179473 1.000000 0:0:0 2021-09-13 16:31:45 [ss-diag] 7 320798 319050 0 0 475 59 31135300 208 128175236 13577644 1.000000 4.520079 1.000000 0:0:0 2021-09-13 16:32:00 [gsi-test] [8] Stop and verify... OK 2021-09-13 16:32:00 [gsi-test] [9] Stop and verify... 2021-09-13T16:32:00.904-07:00 [Info] memstats {"Alloc":375643752, "TotalAlloc":2705433970248, "Sys":976251128, "Lookups":0, "Mallocs":152785627,"Frees":152758810, "HeapAlloc":375643752, "HeapSys":935100416, "HeapIdle":556687360, "HeapInuse":378413056,"HeapReleased":395206656, "HeapObjects":26817,"GCSys":30896128, "LastGC":1631575920824657816,"PauseTotalNs":106726387684, "PauseNs":[74971351, 3952199, 1445470, 1188584, 1483820, 1276468, 1665199, 861720, 1427269, 862071, 2850565, 1180977, 3389047, 108543927, 1141394, 3235663, 3593965, 1257336, 3598735, 2271349, 1155150, 1352007, 30928698, 52129975, 4874787, 4223236, 1336279, 6784672, 108654175, 178016956, 1248780, 8248061, 862728, 1493914, 832979, 4958482, 1086952, 5889111, 4095611, 2961096, 87149766, 1204809, 1597167, 1317713, 76244719, 1545167, 3947955, 2750175, 1310917, 5121326, 484344, 986265, 2950417, 2062222, 1486144, 95495721, 1718243, 76396022, 2331926, 7430120, 4187627, 938831, 2112565, 1334349, 4622679, 1882281, 1768360, 752617, 1663339, 1423294, 45555734, 1287618, 5161725, 3942707, 1398271, 1297331, 1527123, 2675569, 918439, 2052746, 1504300, 3993774, 24746690, 1233119, 1278431, 3275051, 1522141, 1267371, 1229154, 1543687, 1470360, 950650, 5433414, 2109158, 1652232, 945694, 56569461, 2081962, 1153145, 8004413, 1370465, 973008, 1071507, 789252, 1315188, 1429368, 638769, 1232035, 308740, 1638279, 1352065, 1327329, 850981, 1509941, 1221542, 612118, 1955123, 1360806, 880491, 1049096, 1847880, 1545043, 1393111, 1510362, 1241617, 1377336, 1243728, 1561149, 1852253, 1161104, 1257648, 1779170, 1178588, 1690128, 1030321, 1366901, 1466266, 1329453, 1050495, 1835459, 1691595, 1344824, 1727862, 1890869, 1589094, 1388895, 1266871, 1417325, 643453, 809006, 1382553, 2169651, 1986577, 1815905, 1701988, 1723741, 1281580, 1070376, 1714094, 1326040, 1550986, 1382741, 1395971, 1299032, 298953, 531232, 824269, 1460040, 1425183, 1064003, 1026861, 1551601, 926112, 1052396, 776175, 1046828, 1026715, 1352216, 990107, 1399179, 893536, 1178935, 1636233, 1256435], "NumGC":15170} Allocated: 2285932216, active: 2287710208, metadata: 21714000 (n_thp 0), resident: 2454683648, mapped: 2470060032, retained: 3718635520 2021-09-13 16:32:01 [gsi-test] [9] Stop and verify... OK 2021-09-13 16:32:01 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper started 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : all deamons started 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : all deamons started 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : all deamons started 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:32:01 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper started 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : all deamons started 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : all deamons started 2021-09-13 16:32:01 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper started 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : all deamons started 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : all deamons started 2021-09-13 16:32:01 [gsi-test] [] Stop and verify... OK 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:31:45 [ss-diag] 8 99069 97393 0 0 343 1 22482964 4 110074932 196780 1.000000 1.179473 1.000000 0:0:0 2021-09-13 16:31:45 [ss-diag] 9 308679 307017 0 0 529 66 34674892 208 122534796 13577804 1.000000 1.179473 1.000000 0:0:0 Mon Sep 13 16:32:14 PDT 2021 RSS = 2965620 2021-09-13T16:32:31.457-07:00 [Info] memstats {"Alloc":451841696, "TotalAlloc":2745790724584, "Sys":976251128, "Lookups":0, "Mallocs":156266631,"Frees":156239035, "HeapAlloc":451841696, "HeapSys":933822464, "HeapIdle":479232000, "HeapInuse":454590464,"HeapReleased":404897792, "HeapObjects":27596,"GCSys":30994432, "LastGC":1631575951456480367,"PauseTotalNs":108772908953, "PauseNs":[1187989, 1584548, 915363, 708628, 1982763, 1519369, 1367942, 1519057, 1142405, 1442307, 1260963, 1825351, 1100940, 132856351, 3168345, 2513612, 3037727, 140516999, 672970, 4177169, 77600571, 1258773, 1114394, 3488196, 1305451, 1860112, 1957435, 2216430, 10054012, 38922894, 1288233, 2651679, 5739503, 778264, 1537541, 883140, 3122860, 1059686, 4095311, 5920422, 1568418, 712905, 2039851, 55639162, 6969548, 3033941, 775329, 981096, 4167801, 1564597, 769889, 9143824, 1437675, 1439079, 2261954, 126202963, 1685809, 918455, 1686886, 921962, 2171145, 1179756, 60026151, 198717449, 998557, 7800394, 1559223, 1229077, 4563027, 3682240, 1194270, 3173921, 4030942, 1079013, 698469, 913826, 1230759, 95070609, 192165251, 988828, 4412339, 23591755, 1694799, 1913738, 1045942, 1343723, 3496902, 1302354, 1247268, 1127034, 1004833, 5938194, 956351, 1660400, 1416939, 1061898, 944306, 1079327, 1664656, 4162158, 972093, 1633897, 1170526, 1647333, 10024145, 1796242, 1430717, 1329599, 870122, 5735383, 4635849, 638891, 2112812, 1100141, 3509688, 6623211, 4730157, 5566407, 5662669, 3198082, 1223576, 862942, 100759617, 1273820, 1827944, 3787663, 114613196, 6960268, 1694646, 5331050, 1781914, 1339831, 3089457, 106428667, 5735117, 1468017, 1893950, 3875351, 2011409, 591266, 1063658, 4547154, 1802603, 2415530, 2836775, 1050077, 3935828, 1283587, 1117373, 1982332, 2501934, 4285832, 1153756, 2598895, 644615, 4131456, 3039069, 1426414, 5276370, 2451279, 1390273, 101190371, 1524735, 1811530, 1891837, 1765700, 1326401, 22769805, 92934522, 619221], "NumGC":15340} Allocated: 2347910080, active: 2349740032, metadata: 21714000 (n_thp 0), resident: 2546388992, mapped: 2561765376, retained: 3626930176 2021-09-13 16:32:32 [gsi-test] [] QuotaTuner: 2079947910 -> 1871953119 2021-09-13 16:32:38 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: starting... frag 95, data: 6081716, used: 131469230 log:(3987951780 - 4119818240) head:3987951780 2021-09-13 16:32:38 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: completed... frag 98, data: 3393420, used: 183344598, disk: 197496586, relocated: 0, retries: 0, skipped: 9591300, cleaned:1095733248, log:(5083685028 - 5267423232), disk:(5069926646-5267423232), activeFragRatio:30, activeMaxFragRatio:80, run:289 duration:2998 ms elapsed: 299954 ms 2021-09-13T16:33:01.757-07:00 [Info] memstats {"Alloc":463486200, "TotalAlloc":2781817997624, "Sys":976251128, "Lookups":0, "Mallocs":159404124,"Frees":159373115, "HeapAlloc":463486200, "HeapSys":934445056, "HeapIdle":468377600, "HeapInuse":466067456,"HeapReleased":404070400, "HeapObjects":31009,"GCSys":30961664, "LastGC":1631575981748674039,"PauseTotalNs":111751967221, "PauseNs":[3717717, 2972210, 58481730, 1367029, 1032873, 3522138, 2414205, 2929072, 660685, 152154271, 71492746, 3208627, 7900418, 1269984, 1374176, 2221150, 1090668, 106231513, 183158104, 4499790, 1623372, 2095932, 119935555, 4752861, 1750487, 1643508, 115157078, 1696012, 1997467, 964524, 1328160, 4185665, 7090644, 1388835, 1551554, 4153767, 958806, 1756723, 1094951, 2960851, 1671051, 5679578, 1341136, 939931, 5924199, 2007678, 1686871, 1108790, 1817419, 3908386, 134070916, 192159381, 1685065, 3608746, 146956741, 154557165, 3803944, 3257545, 658102, 1017508, 1865037, 154680809, 1277427, 1791717, 2790317, 2402777, 5265704, 10178529, 10490686, 6230449, 9682547, 4398833, 24764851, 12032389, 5797654, 17558692, 9935123, 3039241, 27781109, 8631216, 152870354, 16798413, 3117546, 8706070, 5536971, 10539890, 8203028, 2298790, 13607050, 15029582, 221061267, 6939750, 15466951, 5299464, 8319494, 16076244, 814786, 27072430, 21251076, 9591573, 191151346, 5658543, 1967081, 645770, 2257150, 2241095, 1064071, 1276516, 1462431, 584972, 1579707, 1505957, 1917928, 755495, 2215450, 1376154, 1930714, 974137, 1380969, 1276086, 2001058, 3974933, 1359514, 823861, 137859653, 2201996, 2143525, 1803932, 1289052, 4296831, 1524894, 909161, 617679, 2724958, 1032915, 1929770, 2563672, 104106144, 2005115, 1207073, 1847970, 3682251, 1480951, 1751944, 5568487, 1445374, 3163300, 7384911, 6293926], "NumGC":15489} Allocated: 2525780432, active: 2527547392, metadata: 21736400 (n_thp 0), resident: 2740555776, mapped: 2755907584, retained: 3432787968 2021-09-13 16:33:04 [ss-workload] instance inse% dele% read% 2021-09-13 16:33:04 [ss-workload] 0 14% 14% 70% 2021-09-13 16:33:04 [ss-workload] 1 14% 14% 70% Mon Sep 13 16:33:14 PDT 2021 RSS = 3409100 2021-09-13 16:33:04 [ss-workload] 2 14% 14% 70% 2021-09-13 16:33:04 [ss-workload] 3 14% 14% 70% 2021-09-13 16:33:04 [ss-workload] 4 3% 3% 92% 2021-09-13 16:33:04 [ss-workload] 5 3% 3% 92% 2021-09-13 16:33:04 [ss-workload] 6 3% 3% 92% 2021-09-13 16:33:04 [ss-workload] 7 18% 18% 62% 2021-09-13 16:33:15 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 97, data: 401688, used: 16850886 log:(0 - 17723392) head:0 2021-09-13 16:33:15 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 96, data: 532820, used: 16355036, disk: 18599470, relocated: 0, retries: 0, skipped: 1119, cleaned:16535844, log:(16535844 - 32890880), disk:(14291410-32890880), activeFragRatio:30, activeMaxFragRatio:80, run:21 duration:430 ms elapsed: 305085 ms 2021-09-13 16:33:04 [ss-workload] 8 18% 18% 62% 2021-09-13 16:33:04 [ss-workload] 9 18% 18% 62% 2021-09-13 16:33:04 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:33:04 [ss-diag] 0 64167 62293 0 0 612 2 40115376 5 123127664 262416 1.000000 7.840963 1.000000 0:0:0 2021-09-13 16:33:04 [ss-diag] 1 62307 60549 0 0 368 1 24121664 4 115454428 196764 1.000000 5.747363 1.000000 0:0:0 2021-09-13 16:33:19 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoveryCleaner: starting... frag 97, data: 401688, used: 16981755 log:(0 - 17588224) head:0 2021-09-13 16:33:19 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoveryCleaner: completed... frag 99, data: 139424, used: 16563828, disk: 17882938, relocated: 0, retries: 0, skipped: 717, cleaned:11849984, log:(11849984 - 28676096), disk:(10793158-28676096), activeFragRatio:30, activeMaxFragRatio:80, run:17 duration:58 ms elapsed: 324928 ms 2021-09-13 16:33:04 [ss-diag] 2 62959 61028 0 0 333 1 21827484 4 126800576 196780 1.000000 1.050263 1.000000 0:0:0 2021-09-13 16:33:04 [ss-diag] 3 61418 59546 0 0 363 1 23793924 5 122995656 262368 1.000000 1.050263 1.000000 0:0:0 2021-09-13 16:33:04 [ss-diag] 4 3641803 3640218 0 0 5157 659 338031036 208 117488968 13577836 1.000000 2.449220 1.000000 0:0:0 2021-09-13 16:33:04 [ss-diag] 5 3678163 3676549 0 0 3715 489 243510820 208 119387596 13578028 1.000000 1.050263 1.000000 0:0:0 2021-09-13 16:33:04 [ss-diag] 6 3709463 3707873 0 0 3961 508 259635628 208 117816044 13577692 1.000000 1.050263 1.000000 0:0:0 2021-09-13 16:33:04 [ss-diag] 7 352234 350528 0 0 334 42 21893032 208 125430140 13577644 1.000000 4.958354 1.000000 0:0:0 2021-09-13 16:33:04 [ss-diag] 8 104556 102832 0 0 364 1 23859472 6 113352380 327956 1.000000 1.050263 1.000000 0:0:0 2021-09-13 16:33:04 [ss-diag] 9 339419 337747 0 0 366 44 23990568 208 123193812 13577804 1.000000 1.050263 1.000000 0:0:0 2021-09-13T16:33:32.831-07:00 [Info] memstats {"Alloc":372825024, "TotalAlloc":2814516247776, "Sys":976251128, "Lookups":0, "Mallocs":162295924,"Frees":162262361, "HeapAlloc":372825024, "HeapSys":933920768, "HeapIdle":558555136, "HeapInuse":375365632,"HeapReleased":387276800, "HeapObjects":33563,"GCSys":31027200, "LastGC":1631576012745159392,"PauseTotalNs":115068968723, "PauseNs":[1293059, 3235652, 1998128, 5100551, 3935480, 1795983, 928141, 1230511, 86594572, 1151136, 1345176, 819830, 5327298, 1992216, 1587603, 10107007, 3839931, 155121010, 17811325, 13708347, 23729918, 23845439, 12699427, 15652863, 19034394, 5386983, 7758383, 17537495, 22479019, 8959590, 2037900, 13122729, 19129940, 22361192, 23898876, 24130758, 21710675, 34088885, 20711741, 29226908, 31111012, 11985807, 4580756, 23893308, 19324492, 16444012, 88814685, 176814369, 20077385, 9094618, 13727093, 12602223, 25652100, 22160204, 35241728, 142915886, 29800605, 25699591, 105935090, 230216161, 1656942, 5424102, 6945131, 5539350, 1700144, 10285042, 1651207, 4281664, 1233407, 4773239, 1439722, 2055033, 70098995, 42762131, 105046844, 11344625, 96209436, 8568348, 149472934, 9465419, 19861101, 16199464, 13983418, 14398142, 15973315, 3849539, 1199609, 125886763, 104675423, 3293079, 60770069, 5334776, 3230813, 17111826, 12456666, 8428389, 18145941, 16034183, 9771795, 16361641, 3769826, 238899123, 15309483, 27703561, 13991140, 3560124, 11576249, 1347013, 2649280, 1634006, 3308293, 975554, 1037756, 1062492, 1397358, 1156308, 1611069, 3824555, 3290711, 56446565, 845092, 967023, 3633860, 1043527, 4743237, 2571465, 128902172, 6092592, 1478906, 4487801, 1192076, 7056422], "NumGC":15621} Allocated: 2914320704, active: 2916073472, metadata: 21737808 (n_thp 0), resident: 3078193152, mapped: 3093544960, retained: 3095150592 2021-09-13T16:34:03.136-07:00 [Info] memstats {"Alloc":301467704, "TotalAlloc":2850513712496, "Sys":976251128, "Lookups":0, "Mallocs":165404702,"Frees":165382178, "HeapAlloc":301467704, "HeapSys":934248448, "HeapIdle":629383168, "HeapInuse":304865280,"HeapReleased":391659520, "HeapObjects":22524,"GCSys":30896128, "LastGC":1631576043078009760,"PauseTotalNs":118657294162, "PauseNs":[2657313, 205671736, 6957528, 5823926, 6174260, 872235, 6326870, 1594480, 912257, 5703635, 7101135, 1039189, 2005553, 1777355, 1633187, 1475455, 250526783, 5986265, 2970373, 8240572, 486288, 1304658, 813292, 922124, 899290, 2010847, 5340431, 4479546, 2296433, 1949498, 96312976, 1025265, 1893638, 109766163, 3044785, 786429, 1129348, 999495, 892302, 4358503, 718309, 21338594, 88673348, 2005414, 751038, 167663844, 1011451, 27369061, 180500424, 1549403, 3187999, 1164173, 1529332, 72031987, 1664157, 1369010, 4809356, 184640240, 3820713, 1448347, 4113565, 1420839, 2308112, 2657233, 1273604, 2198413, 4369323, 8105305, 4632844, 7070043, 956695, 201110790, 99174652, 1177334, 6014333, 2117558, 2446758, 2620541, 1778874, 1514746, 1582446, 1897172, 4025379, 10888054, 1338546, 1604047, 208043938, 2412849, 1933751, 1709013, 3005113, 1476811, 1485643, 2069110, 2169007, 1195849, 1970829, 7475422, 931451, 1002280, 589937, 1237499, 1929039, 1581821, 4512265, 1865069, 2461600, 1551918, 1381993, 1128246, 927911, 5440195, 162975981, 5955780, 1610669, 4105247, 5049638, 1616021, 1314275, 1735152, 246589781, 1067960, 6034229, 4655578, 1444346, 1847977, 3834284, 1779140, 1352754, 83629658, 1814773, 214380703, 143824750, 1772469, 2824171, 120410958, 186081038, 1886840, 4463095, 5932173, 112602016, 3894496, 2820656, 65466505, 1303649, 637524, 6486018, 1439509, 392243], "NumGC":15770} Allocated: 2985098864, active: 2986819584, metadata: 21743312 (n_thp 0), resident: 3153055744, mapped: 3171024896, retained: 3017670656 Mon Sep 13 16:34:14 PDT 2021 RSS = 3447472 2021-09-13 16:34:23 [plasma] [i] Plasma: mvccPurger: starting... purge_ratio 11.00962 2021-09-13 16:34:29 [ss-workload] instance inse% dele% read% 2021-09-13 16:34:29 [ss-workload] 0 14% 14% 70% 2021-09-13 16:34:29 [ss-workload] 1 14% 14% 70% 2021-09-13 16:34:29 [ss-workload] 2 14% 14% 70% 2021-09-13 16:34:29 [ss-workload] 3 14% 14% 70% 2021-09-13 16:34:29 [ss-workload] 4 3% 3% 92% 2021-09-13 16:34:29 [ss-workload] 5 3% 3% 92% 2021-09-13 16:34:29 [ss-workload] 6 3% 3% 92% 2021-09-13 16:34:29 [ss-workload] 7 18% 18% 62% 2021-09-13T16:34:33.453-07:00 [Info] memstats {"Alloc":434711296, "TotalAlloc":2893067111664, "Sys":976251128, "Lookups":0, "Mallocs":169154672,"Frees":169119012, "HeapAlloc":434711296, "HeapSys":934313984, "HeapIdle":497082368, "HeapInuse":437231616,"HeapReleased":390291456, "HeapObjects":35660,"GCSys":30896128, "LastGC":1631576073275215325,"PauseTotalNs":119591678937, "PauseNs":[4113506, 1201036, 1645594, 4111113, 8988814, 434005, 1084434, 5754440, 498772, 4220159, 5992610, 855203, 756319, 1861115, 1149438, 719863, 3245183, 1805216, 8567442, 1658479, 982848, 8656219, 2448929, 2680251, 5359681, 4469940, 1726782, 4201385, 3733166, 7161457, 1837098, 856106, 1094325, 1338404, 2738391, 1413301, 154200252, 1349600, 896201, 620609, 773584, 1161024, 1291368, 6735809, 1424714, 1346702, 750574, 1112395, 4319874, 1113892, 3773968, 1749238, 2363939, 1710939, 47488405, 3863582, 1504529, 2434225, 5373726, 5689995, 3317853, 3773772, 3022139, 2940380, 4847107, 5842110, 1365618, 1096113, 1301732, 953475, 2319449, 1343901, 870931, 2585878, 1466443, 5175325, 1035451, 1171911, 1144833, 2613162, 1605933, 3465379, 1101407, 1245198, 1266297, 1340253, 1741047, 5449437, 3965789, 1759751, 4734281, 1198738, 2965432, 1290878, 1166764, 2126014, 843267, 5806409, 821370, 514604, 4036523, 1696641, 1748885, 2116012, 4520109, 1378162, 2423401, 3832589, 1070910, 703399, 830268, 1111777, 3611314, 72835959, 783601, 4090019, 960872, 771243, 2421754, 17472850, 8137200, 1286197, 4029958, 4566964, 1638799, 2636111, 1700232, 1042117, 3398254, 1233345, 1229017, 763956, 3263051, 1288287, 1452929, 1529469, 1322474, 1353385, 1122328, 3335486, 1304168, 1324216, 5197062, 767604, 1872967, 1444435, 1682509, 2627685, 5945433, 6128143, 1191747, 520669, 2072469, 3920325, 2926829, 1801446, 1039683, 1303956, 1758981, 1683124, 4470807, 1972959, 870232, 1790734, 3815369, 8674303, 7509938, 838784, 2588325, 4275487, 2515089, 1331257, 1493496, 209772499, 4706338, 1493861], "NumGC":15946} Allocated: 6004552280, active: 6006308864, metadata: 25992928 (n_thp 0), resident: 6202322944, mapped: 6226001920, retained: 1720107008 2021-09-13 16:34:29 [ss-workload] 8 18% 18% 62% 2021-09-13 16:34:29 [ss-workload] 9 18% 18% 62% 2021-09-13 16:34:29 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:34:29 [ss-diag] 0 66622 64506 0 0 351 1 23007348 5 138999224 262416 1.000000 8.822438 1.000000 0:0:0 2021-09-13 16:34:29 [ss-diag] 1 64693 62785 0 0 365 1 23925020 4 125292052 196764 1.000000 11.026228 1.000000 0:0:0 2021-09-13 16:34:29 [ss-diag] 2 66129 64443 0 0 335 1 21958580 4 110730812 196780 1.000000 1.073233 1.000000 0:0:0 2021-09-13 16:34:38 [plasma] [i] Plasma: Adjusting SMO limits: Avg - Item Size 65548 Avg Page Size 23217101: Old MaxPageItems 400 MinPageItems 25 MaxDeltaChainLen 200 -> New MaxPageItems 4 MinPageItems 1 MaxDeltaChainLen 2 2021-09-13 16:34:29 [ss-diag] 3 63449 61235 0 0 355 1 23269540 5 145427072 262368 1.000000 1.073233 1.000000 0:0:0 2021-09-13 16:34:29 [ss-diag] 4 4002578 4000988 0 0 5904 745 386995392 208 117816684 13577836 1.000000 2.759957 1.000000 0:0:0 2021-09-13 16:34:29 [ss-diag] 5 4048046 4046464 0 0 5952 771 390141696 208 117289740 13578028 1.000000 1.073233 1.000000 0:0:0 2021-09-13 16:34:29 [ss-diag] 6 4078699 4077099 0 0 5124 659 335867952 208 118469684 13577692 1.000000 1.073233 1.000000 0:0:0 2021-09-13 16:34:29 [ss-diag] 7 384862 383168 0 0 530 64 34740440 208 124643532 13577644 1.000000 4.601993 1.000000 0:0:0 2021-09-13 16:34:47 [plasma] [i] Plasma: mvccPurger: finished... purge_ratio 7.41063 2021-09-13 16:34:29 [ss-diag] 8 200235 198697 0 0 11427 33 749016996 21 102137840 1311840 1.000000 1.073233 1.000000 0:0:0 2021-09-13 16:34:29 [ss-diag] 9 373920 372208 0 0 734 87 48112232 208 125812084 13577804 1.000000 1.073233 1.000000 0:0:0 2021-09-13T16:35:03.904-07:00 [Info] memstats {"Alloc":374893760, "TotalAlloc":2925656899912, "Sys":976251128, "Lookups":0, "Mallocs":172102968,"Frees":172074586, "HeapAlloc":374893760, "HeapSys":934445056, "HeapIdle":556654592, "HeapInuse":377790464,"HeapReleased":400998400, "HeapObjects":28382,"GCSys":30896128, "LastGC":1631576103216120341,"PauseTotalNs":121707554491, "PauseNs":[2243367, 3713541, 5053406, 1561125, 5557596, 7167401, 901427, 1325296, 1046804, 814152, 1231541, 3199929, 645709, 1250261, 1848689, 1407521, 2381356, 1712466, 146601787, 6732065, 4761238, 4436608, 811107, 4006284, 1517369, 33706426, 1911586, 3235752, 4140197, 5277214, 6066034, 7231574, 3309863, 11952212, 8598007, 2774283, 2628288, 20546610, 12263923, 187701157, 15023069, 1364739, 127513953, 1720736, 10180910, 6986522, 162912670, 3583124, 8479618, 13772445, 14400954, 1478125, 13198493, 21857376, 5727778, 2959024, 1864871, 2323391, 6740221, 15258379, 17230201, 6361780, 17614170, 2823389, 8140854, 16685759, 7633176, 5138045, 13702312, 5495883, 237564142, 9530917, 10189368, 12389582, 9895304, 12863695, 20959569, 8293623, 11629001, 6418747, 16279346, 545548, 7814259, 1425267, 873850, 2010498, 633806, 1774159, 1607141, 4634610, 9401619, 6375922, 10502359, 8024541, 6709175, 11103639, 89985561, 3761001, 1648200, 106584810, 4408331, 3405013, 15254995, 1006779, 15550843, 184262398, 1185754, 905362, 1316556, 86877110, 3022694, 1755685, 1157837, 1187107, 2927088, 4215791, 2586060, 2798691, 1363796, 1355574, 62396328, 918688, 3343525, 3247590, 3567543, 819535, 1779311, 2094907, 474496, 1538364, 4103345, 415964, 1760097], "NumGC":16079} Allocated: 2450271592, active: 2452021248, metadata: 32305760 (n_thp 0), resident: 2725306368, mapped: 2742673408, retained: 8022007808 Mon Sep 13 16:35:14 PDT 2021 RSS = 2976560 2021-09-13 16:35:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 158, "num_active_sctxs": 81, "num_free_sctxs": 114, "num_trim_sctxs": 9, "num_wctxs": 238, "num_free_wctxs": 185, "num_active_wctxs": 53, "num_trim_wctxs": 1073, "num_readers": 67, "num_writers": 98, "cpu": 8.63800, "total_core": 16, "threads": 54, "assigned_quota": 1871953119, "current_quota": 1871953119, "memory_resident": 0, "memory_jemalloc": 2532859904, "memory_jemalloc_allocated": 2455378944, "memory_in_use": 2532859904, "memory_stats_size_total": 2068360817, "memory_stats_size_global": 1306198464, "memory_stats_size_page": 1228899660, "memory_stats_size_index": 77465460, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 208144, "reclaim_pending": 2819276, "buf_memused": 759176421, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 2, "num_pending_smr": 0, "num_pages": 1191, "items_count": 2080, "total_records": 17562, "total_ops": 863738, "compacts": 1977741, "purges": 413, "lss_data_size": 238935280, "lss_used_space": 1135067446, "lss_fragmentation": 78, "lss_num_reads": 733988, "lss_read_bs": 139464794568, "lss_blk_read_bs": 142461091840, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 140473835520, "bytes_incoming": 49798519848, "write_amp_avg": 1.52, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 277 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 17562, "resident_ratio": 100.00000, "cleaner_num_reads": 733988, "cleaner_read_bs": 139464794568, "cleaner_blk_read_bs": 142461091840, "recovery_num_lssctx": 5, "recovery_num_sctxs": 29, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 9, "recovery_num_trim_sctxs": 9, "recovery_buf_memused": 5242880, "recovery_data_size": 29259556, "recovery_used_space": 257206580, "recovery_lss_blk_read_bs": 81016782848, "recovery_lss_read_bs": 78706116512, "recovery_lss__num_reads": 611359, "recovery_bytes_written": 79136378880, "recovery_lss_frag": 88, "recovery_cleaner_num_reads": 611359, "recovery_cleaner_read_bs": 78706116512, "recovery_cleaner_blk_read_bs": 81016782848, "mvcc_purge_ratio": 8.44327 }2021-09-13T16:35:33.960-07:00 [Info] memstats {"Alloc":325153512, "TotalAlloc":2933630550096, "Sys":976251128, "Lookups":0, "Mallocs":173152233,"Frees":173122893, "HeapAlloc":325153512, "HeapSys":934477824, "HeapIdle":606461952, "HeapInuse":328015872,"HeapReleased":440066048, "HeapObjects":29340,"GCSys":30994432, "LastGC":1631576133795272156,"PauseTotalNs":121887876581, "PauseNs":[4544789, 1153804, 7214060, 1932177, 433303, 2906182, 1111512, 1143879, 503543, 3058445, 5152192, 496942, 2178065, 2796387, 761509, 3799981, 7750855, 7578130, 929680, 103083607, 471877, 5886153, 1226805, 826838, 903192, 585033, 667996, 1433321, 1151208, 1275531, 471937, 423406, 1036533, 892025, 1032517, 3508676], "NumGC":16115} Allocated: 2463525632, active: 2465259520, metadata: 32305760 (n_thp 0), resident: 2543312896, mapped: 2560679936, retained: 8204001280 2021-09-13 16:35:53 [ss-workload] instance inse% dele% read% 2021-09-13 16:35:53 [ss-workload] 0 14% 14% 70% 2021-09-13 16:35:53 [ss-workload] 1 14% 14% 70% 2021-09-13 16:36:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: starting... frag 41, data: 38830125, used: 65914952 log:(5293974248 - 5360852992) head:5289330009 2021-09-13 16:36:01 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: completed... frag 30, data: 49997211, used: 71880834, disk: 81131140, relocated: 4958884, retries: 6650, skipped: 86652, cleaned:1575631595, log:(6869605843 - 6942199808), disk:(6861068668-6942199808), activeFragRatio:30, activeMaxFragRatio:80, run:235 duration:99052 ms elapsed: 300141 ms 2021-09-13 16:35:53 [ss-workload] 2 14% 14% 70% 2021-09-13 16:36:03 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 94, data: 3155424, used: 56090396 log:(41117061598 - 41173680128) head:41117061598 2021-09-13 16:36:03 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 90, data: 3942216, used: 39894550, disk: 41471860, relocated: 58476028, retries: 380586, skipped: 14441017, cleaned:10555551476, log:(51672613074 - 51713032192), disk:(51671560332-51713032192), activeFragRatio:30, activeMaxFragRatio:80, run:207 duration:256651 ms elapsed: 300405 ms 2021-09-13 16:35:53 [ss-workload] 3 14% 14% 70% 2021-09-13T16:36:04.012-07:00 [Info] memstats {"Alloc":427200504, "TotalAlloc":2944237262552, "Sys":976251128, "Lookups":0, "Mallocs":174342140,"Frees":174311163, "HeapAlloc":427200504, "HeapSys":934019072, "HeapIdle":504020992, "HeapInuse":429998080,"HeapReleased":423362560, "HeapObjects":30977,"GCSys":30896128, "LastGC":1631576163871856643,"PauseTotalNs":122068011985, "PauseNs":[3497853, 429748, 684744, 436309, 944856, 2311554, 3244678, 805955, 625279, 2646091, 942543, 4176830, 1712619, 358449, 457113, 1312999, 1116639, 2152808, 89824638, 340649, 482373, 4401064, 1070468, 2346096, 4430241, 1138538, 1559032, 1348773, 364708, 1354188, 602510, 541093, 362761, 1579360, 1267228, 886355, 1727626, 1306894, 1870728, 1678524, 1164271, 20986122, 5741520, 1112037, 1481423, 1309117], "NumGC":16161} Allocated: 2577019984, active: 2578751488, metadata: 32305760 (n_thp 0), resident: 2771427328, mapped: 2789580800, retained: 7975100416 2021-09-13 16:35:53 [ss-workload] 4 3% 3% 92% 2021-09-13 16:35:53 [ss-workload] 5 3% 3% 92% 2021-09-13 16:35:53 [ss-workload] 6 3% 3% 92% 2021-09-13 16:35:53 [ss-workload] 7 18% 18% 62% 2021-09-13 16:35:53 [ss-workload] 8 18% 18% 62% 2021-09-13 16:36:08 [gsi-test] [] QuotaTuner: 1871953119 -> 1684757807 2021-09-13 16:35:53 [ss-workload] 9 18% 18% 62% 2021-09-13 16:35:53 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:35:53 [ss-diag] 0 68835 66998 0 0 186 1 12191928 5 120700780 262416 1.000000 2.587121 1.000000 0:0:0 2021-09-13 16:35:53 [ss-diag] 1 66532 64484 0 0 371 1 24318308 4 134474212 196764 1.000000 1.823170 1.000000 0:0:0 2021-09-13 16:35:53 [ss-diag] 2 66859 65033 0 0 335 1 21958580 4 119913580 196780 1.000000 2403.716116 1.000000 0:0:0 2021-09-13 16:35:53 [ss-diag] 3 64728 62738 0 0 375 1 24580500 5 130735040 262368 1.000000 2403.716116 1.000000 0:0:0 2021-09-13 16:35:53 [ss-diag] 4 4306083 4304465 0 0 3719 490 243773012 208 119652124 13577836 1.000000 5.151103 1.000000 0:0:0 2021-09-13 16:35:53 [ss-diag] 5 4185031 4183437 0 0 3189 398 209032572 208 118076860 13578028 1.000000 2403.716116 1.000000 0:0:0 2021-09-13 16:35:53 [ss-diag] 6 4213824 4212234 0 0 2827 351 185304196 208 117814188 13577692 1.000000 2403.716116 1.000000 0:0:0 Mon Sep 13 16:36:14 PDT 2021 RSS = 3188380 2021-09-13 16:35:53 [ss-diag] 7 412602 410914 0 0 425 52 27857900 208 124251348 13577644 1.000000 5.945420 1.000000 0:0:0 2021-09-13 16:36:15 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 40, data: 18366736, used: 30813718 log:(36147478948 - 36178423808) head:36146426136 2021-09-13 16:36:15 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 45, data: 16334208, used: 30023231, disk: 31076282, relocated: 81223432, retries: 38474, skipped: 6090202, cleaned:8804851432, log:(44952330380 - 44982353920), disk:(44951277638-44982353920), activeFragRatio:30, activeMaxFragRatio:80, run:282 duration:232638 ms elapsed: 300116 ms 2021-09-13 16:35:53 [ss-diag] 8 216295 214594 0 0 599 70 39263252 207 125030228 13512216 1.000000 2403.716116 1.000000 0:0:0 2021-09-13 16:35:53 [ss-diag] 9 381876 380197 0 0 377 46 24711596 208 123651680 13577804 1.000000 2403.716116 1.000000 0:0:0 2021-09-13 16:36:23 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: starting... frag 91, data: 10610603, used: 124285463 log:(18861672602 - 18986278912) head:18859101251 2021-09-13 16:36:23 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: completed... frag 88, data: 7754304, used: 69185301, disk: 75898872, relocated: 3460426, retries: 86771, skipped: 20467, cleaned:4577643993, log:(23439316595 - 23508824064), disk:(23432925192-23508824064), activeFragRatio:30, activeMaxFragRatio:80, run:49 duration:291030 ms elapsed: 304275 ms 2021-09-13 16:36:33 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: starting... frag 55, data: 59806187, used: 133512999 log:(3537244377 - 3670757376) head:3516437252 2021-09-13 16:36:33 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: completed... frag 31, data: 66936690, used: 97763818, disk: 112712112, relocated: 3429, retries: 2448, skipped: 117, cleaned:1228361003, log:(4765605380 - 4863369216), disk:(4750657104-4863369216), activeFragRatio:30, activeMaxFragRatio:80, run:19 duration:60545 ms elapsed: 334774 ms 2021-09-13T16:36:34.487-07:00 [Info] memstats {"Alloc":413557824, "TotalAlloc":2983961473856, "Sys":976251128, "Lookups":0, "Mallocs":177824556,"Frees":177793313, "HeapAlloc":413557824, "HeapSys":933691392, "HeapIdle":517545984, "HeapInuse":416145408,"HeapReleased":407945216, "HeapObjects":31243,"GCSys":30994432, "LastGC":1631576194442375773,"PauseTotalNs":124278068615, "PauseNs":[4842801, 1229536, 455213, 1766568, 1011712, 2507171, 3349887, 25070318, 5977697, 100295707, 1414662, 4667719, 1885408, 8253189, 1125283, 6994166, 10894718, 1097561, 84373169, 3660530, 3304255, 3199917, 2056780, 1723728, 3449459, 1911532, 2813821, 1538972, 1737367, 6900690, 5650057, 885718, 2083597, 2324080, 1236319, 2486526, 6282429, 1661266, 2322483, 1208496, 2198288, 4299993, 4570491, 488036, 2077554, 1852176, 4116843, 1367879, 1026234, 5783079, 1332110, 911286, 3246637, 1365216, 6211889, 4808879, 150161311, 166951557, 5515541, 1291825, 2336977, 976884, 1264188, 641098, 3403622, 3606190, 1529153, 111506457, 1665876, 1773298, 1574478, 1200380, 2022834, 1069468, 13674719, 3032742, 2202722, 8484702, 60294014, 1185293, 4045375, 1399628, 143027708, 1234275, 1970164, 210125666, 255753004, 1882909, 1236083, 1781527, 1255005, 967774, 1350291, 1610090, 5324854, 9040348, 1074708, 926175, 1403622, 1561864, 1548391, 1556784, 997254, 2246067, 1543540, 1107477, 5270601, 2279423, 4228099, 891380, 2684979, 1478995, 3022243, 2681733, 4487793, 1105410, 3670332, 1000666, 712601, 1459014, 974512, 3883588, 841568, 2182098, 1854617, 4961586, 1391995, 1740614, 4930056, 4353834, 2493430, 4445435, 1263501, 5034226, 1966123, 6348052, 2704432, 120538571, 1215723, 986504, 761128, 128702870, 32783947, 904425, 1683544, 917804, 4535361, 4514105, 1810162, 3394286, 3256192, 789952, 42544856, 1693919, 3373802, 1079296, 2424394, 1034285, 1227883, 4428063, 5224259, 1097424, 2493353, 133724179, 829388, 41101010], "NumGC":16327} Allocated: 2668819528, active: 2670546944, metadata: 32305760 (n_thp 0), resident: 2891534336, mapped: 2908901376, retained: 7855779840 2021-09-13 16:36:38 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: starting... frag 64, data: 37582249, used: 104726528 log:(3253066332 - 3357794304) head:3228019792 2021-09-13 16:36:38 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: completed... frag 63, data: 49356544, used: 136122500, disk: 150150076, relocated: 2968, retries: 1869, skipped: 272, cleaned:1291314446, log:(4544380778 - 4680503296), disk:(4530353220-4680503296), activeFragRatio:30, activeMaxFragRatio:80, run:17 duration:67969 ms elapsed: 302231 ms 2021-09-13 16:36:40 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: starting... frag 83, data: 125956877, used: 777619986 log:(32199156891 - 32977776640) head:32178259258 2021-09-13 16:36:40 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: completed... frag 80, data: 119047096, used: 625656798, disk: 663650971, relocated: 229334, retries: 14249, skipped: 7422, cleaned:4991713700, log:(37190870591 - 37816848384), disk:(37153197413-37816848384), activeFragRatio:30, activeMaxFragRatio:80, run:6 duration:267890 ms elapsed: 318611 ms 2021-09-13 16:36:52 [gsi-test] [] { "memory_quota": 1684757807, "count": 28068, "compacts": 142, "purges": 0, "splits": 4, "merges": 0, "inserts": 28068, "deletes": 0, "compact_conflicts": 255, "split_conflicts": 11, "merge_conflicts": 0, "insert_conflicts": 3255, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 133097328, "memory_size_index": 262416, "allocated": 4604787528, "freed": 4471690200, "reclaimed": 4471690200, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 262416, "freed_index": 0, "reclaimed_index": 0, "num_pages": 5, "items_count": 208, "total_records": 2026, "num_rec_allocs": 70157, "num_rec_frees": 68131, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1839801264, "lss_data_size": 71006781, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 100009550, "est_recovery_size": 10349430, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 83920, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 55852, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 9.74038, "currSn": 2056, "gcSn": 2053, "gcSnIntervals": "[0 581 1144 1716 2054]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 4, "num_free_wctxs": 10, "num_trim_wctxs": 105, "num_readers": 6, "num_writers": 4, "page_bytes": 12191928, "page_cnt": 1, "page_itemcnt": 186, "avg_item_size": 65548, "avg_page_size": 12191928, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 134146384, "page_bytes_marshalled":5250954656, "page_bytes_compressed":2858048021, "compression_ratio": 1.83725, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 23008, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 138336397, "lss_data_size": 71539601, "lss_used_space": 101887437, "lss_disk_size": 214654976, "lss_fragmentation": 29, "lss_num_reads": 853, "lss_read_bs": 5186595800, "lss_blk_read_bs": 5203992576, "bytes_written": 5247819776, "bytes_incoming": 1839801264, "write_amp": 2.59, "write_amp_avg": 2.83, "lss_gc_num_reads": 853, "lss_gc_reads_bs": 5186595800, "lss_blk_gc_reads_bs": 5203992576, "lss_gc_status": "frag 50, data: 69080569, used: 138714869 log:(4985217291 - 5123932160) head:4967129990", "lss_head_offset": 5123928335, "lss_tail_offset": 5209391104, "num_sctxs": 20, "num_free_sctxs": 10, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:36:54 [gsi-test] [] { "memory_quota": 1684757807, "count": 27430, "compacts": 139, "purges": 0, "splits": 3, "merges": 0, "inserts": 27430, "deletes": 0, "compact_conflicts": 188, "split_conflicts": 8, "merge_conflicts": 0, "insert_conflicts": 2896, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 118470196, "memory_size_index": 196764, "allocated": 4499679908, "freed": 4381209712, "reclaimed": 4381209712, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 196764, "freed_index": 0, "reclaimed_index": 0, "num_pages": 4, "items_count": 208, "total_records": 1804, "num_rec_allocs": 68558, "num_rec_frees": 66754, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1797981640, "lss_data_size": 63369819, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 154560534, "est_recovery_size": 10042294, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 81947, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 54517, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 8.67308, "currSn": 2062, "gcSn": 2054, "gcSnIntervals": "[0 579 1155 1728 2055]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 11, "num_trim_wctxs": 111, "num_readers": 7, "num_writers": 4, "page_bytes": 24777144, "page_cnt": 1, "page_itemcnt": 378, "avg_item_size": 65548, "avg_page_size": 24777144, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 117355328, "page_bytes_marshalled":5287335476, "page_bytes_compressed":2877834532, "compression_ratio": 1.83726, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12912, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 138315957, "lss_data_size": 63771507, "lss_used_space": 158027508, "lss_disk_size": 211931136, "lss_fragmentation": 59, "lss_num_reads": 790, "lss_read_bs": 5059797291, "lss_blk_read_bs": 5074468864, "bytes_written": 4976660480, "bytes_incoming": 1797981640, "write_amp": 1.82, "write_amp_avg": 2.75, "lss_gc_num_reads": 790, "lss_gc_reads_bs": 5059797291, "lss_blk_gc_reads_bs": 5074468864, "lss_gc_status": "frag 55, data: 62374877, used: 141664251 log:(4799066099 - 4940730368) head:4782682713", "lss_head_offset": 4799066099, "lss_tail_offset": 4940730368, "num_sctxs": 21, "num_free_sctxs": 12, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:36:54 [gsi-test] [] { "memory_quota": 1684757807, "count": 27074, "compacts": 136, "purges": 0, "splits": 3, "merges": 0, "inserts": 27074, "deletes": 0, "compact_conflicts": 237, "split_conflicts": 6, "merge_conflicts": 0, "insert_conflicts": 2921, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 8, "memory_size": 128046876, "memory_size_index": 196780, "allocated": 4470491584, "freed": 4342444708, "reclaimed": 4342444708, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 196780, "freed_index": 0, "reclaimed_index": 0, "num_pages": 4, "items_count": 208, "total_records": 1950, "num_rec_allocs": 68114, "num_rec_frees": 66164, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1774646552, "lss_data_size": 46920332, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 167572614, "est_recovery_size": 405960, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 81100, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 54026, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 9.37500, "currSn": 1916, "gcSn": 1914, "gcSnIntervals": "[0 466 1047 1626 1915]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 2, "num_free_wctxs": 19, "num_trim_wctxs": 107, "num_readers": 9, "num_writers": 9, "page_bytes": 21630840, "page_cnt": 1, "page_itemcnt": 330, "avg_item_size": 65548, "avg_page_size": 21630840, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 128374512, "page_bytes_marshalled":3842060768, "page_bytes_compressed":2091221547, "compression_ratio": 1.83723, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 19904, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 504779241, "lss_data_size": 184926624, "lss_used_space": 680936348, "lss_disk_size": 716648448, "lss_fragmentation": 72, "lss_num_reads": 94487, "lss_read_bs": 43467052657, "lss_blk_read_bs": 44786589696, "bytes_written": 43934756864, "bytes_incoming": 37199014384, "write_amp": 2403.72, "write_amp_avg": 1.02, "lss_gc_num_reads": 94487, "lss_gc_reads_bs": 43467052657, "lss_blk_gc_reads_bs": 44786589696, "lss_gc_status": "frag 80, data: 119047096, used: 625656798 log:(37190870591 - 37816848384) head:37153197413", "lss_head_offset": 37413799232, "lss_tail_offset": 37945741312, "num_sctxs": 68, "num_free_sctxs": 57, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:36:54 [gsi-test] [] { "memory_quota": 1684757807, "count": 26368, "compacts": 133, "purges": 0, "splits": 4, "merges": 0, "inserts": 26368, "deletes": 0, "compact_conflicts": 244, "split_conflicts": 6, "merge_conflicts": 0, "insert_conflicts": 3203, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 137687112, "memory_size_index": 262368, "allocated": 4319756044, "freed": 4182068932, "reclaimed": 4182068932, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 50, "allocated_index": 262368, "freed_index": 0, "reclaimed_index": 0, "num_pages": 5, "items_count": 207, "total_records": 2096, "num_rec_allocs": 65813, "num_rec_frees": 63717, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1728369664, "lss_data_size": 68433097, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 285137904, "est_recovery_size": 537114, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 79054, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 52686, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 10.12560, "currSn": 1902, "gcSn": 1899, "gcSnIntervals": "[0 468 1050 1615 1900]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 15, "num_trim_wctxs": 101, "num_readers": 8, "num_writers": 11, "page_bytes": 24580500, "page_cnt": 1, "page_itemcnt": 375, "avg_item_size": 65548, "avg_page_size": 24580500, "act_max_page_items": 400, "act_min_page_items": 25, "act_max_delta_len": 200, "est_resident_mem": 117223416, "page_bytes_marshalled":3928914168, "page_bytes_compressed":2138484888, "compression_ratio": 1.83724, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 24232, "num_total_smr": 1, "num_active_smr": 6, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 504779241, "lss_data_size": 161855556, "lss_used_space": 700864416, "lss_disk_size": 737218560, "lss_fragmentation": 76, "lss_num_reads": 94489, "lss_read_bs": 43486035008, "lss_blk_read_bs": 44805562368, "bytes_written": 43955326976, "bytes_incoming": 37206880144, "write_amp": 2403.72, "write_amp_avg": 1.02, "lss_gc_num_reads": 94489, "lss_gc_reads_bs": 43486035008, "lss_blk_gc_reads_bs": 44805562368, "lss_gc_status": "frag 80, data: 119047096, used: 625656798 log:(37190870591 - 37816848384) head:37153197413", "lss_head_offset": 37415155316, "lss_tail_offset": 37966311424, "num_sctxs": 68, "num_free_sctxs": 53, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:36:54 [gsi-test] [] { "memory_quota": 1684757807, "count": 216038, "compacts": 669254, "purges": 0, "splits": 207, "merges": 0, "inserts": 216038, "deletes": 0, "compact_conflicts": 246, "split_conflicts": 28, "merge_conflicts": 0, "insert_conflicts": 2419, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 115849300, "memory_size_index": 13577836, "allocated": 337768347980, "freed": 337652498680, "reclaimed": 337643647708, "reclaim_pending": 8850972, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577836, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1560, "num_rec_allocs": 4485735, "num_rec_frees": 4484175, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 14160858824, "lss_data_size": 9613352, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 64089013, "est_recovery_size": 33383948, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 3626464, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 3410426, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.50000, "currSn": 37511, "gcSn": 37494, "gcSnIntervals": "[0 9651 20176 30592 37495]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 5, "num_free_wctxs": 28, "num_trim_wctxs": 133, "num_readers": 11, "num_writers": 16, "page_bytes": 277268040, "page_cnt": 550, "page_itemcnt": 4230, "avg_item_size": 65548, "avg_page_size": 504123, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 132311728, "page_bytes_marshalled":38530069652, "page_bytes_compressed":20993665522, "compression_ratio": 1.83532, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 13504, "num_total_smr": 8, "num_active_smr": 3, "num_pending_smr": 5, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 117643300, "lss_data_size": 13227756, "lss_used_space": 93922513, "lss_disk_size": 180523008, "lss_fragmentation": 85, "lss_num_reads": 491718, "lss_read_bs": 76780252946, "lss_blk_read_bs": 77754195968, "bytes_written": 77087281152, "bytes_incoming": 14160858824, "write_amp": 5.15, "write_amp_avg": 1.69, "lss_gc_num_reads": 491718, "lss_gc_reads_bs": 76780252946, "lss_blk_gc_reads_bs": 77754195968, "lss_gc_status": "frag 84, data: 9256076, used: 60041085 log:(23834665171 - 23895670784) head:23817383357", "lss_head_offset": 23854983773, "lss_tail_offset": 23915991040, "num_sctxs": 36, "num_free_sctxs": 27, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:36:54 [gsi-test] [] { "memory_quota": 1684757807, "count": 212080, "compacts": 652463, "purges": 0, "splits": 207, "merges": 0, "inserts": 212080, "deletes": 0, "compact_conflicts": 280, "split_conflicts": 21, "merge_conflicts": 0, "insert_conflicts": 2197, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 15, "memory_size": 115323252, "memory_size_index": 13578028, "allocated": 329778189692, "freed": 329662866440, "reclaimed": 329662341944, "reclaim_pending": 524496, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13578028, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1552, "num_rec_allocs": 4380422, "num_rec_frees": 4378870, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 13901419840, "lss_data_size": 1365233, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 5688470, "est_recovery_size": 49846958, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 3613219, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 3401139, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.46154, "currSn": 35810, "gcSn": 35796, "gcSnIntervals": "[0 7887 19143 30229 35797]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 25, "num_trim_wctxs": 143, "num_readers": 7, "num_writers": 16, "page_bytes": 247378152, "page_cnt": 480, "page_itemcnt": 3774, "avg_item_size": 65548, "avg_page_size": 515371, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 131130248, "page_bytes_marshalled":8365212784, "page_bytes_compressed":4557671995, "compression_ratio": 1.83541, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 11904, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 504779241, "lss_data_size": 161855556, "lss_used_space": 700864416, "lss_disk_size": 737218560, "lss_fragmentation": 76, "lss_num_reads": 94489, "lss_read_bs": 43486035008, "lss_blk_read_bs": 44805562368, "bytes_written": 43955326976, "bytes_incoming": 37206945692, "write_amp": 2403.72, "write_amp_avg": 1.02, "lss_gc_num_reads": 94489, "lss_gc_reads_bs": 43486035008, "lss_blk_gc_reads_bs": 44805562368, "lss_gc_status": "frag 80, data: 119047096, used: 625656798 log:(37190870591 - 37816848384) head:37153197413", "lss_head_offset": 37415155316, "lss_tail_offset": 37966311424, "num_sctxs": 68, "num_free_sctxs": 54, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:36:54 [gsi-test] [] { "memory_quota": 1684757807, "count": 212108, "compacts": 653875, "purges": 0, "splits": 207, "merges": 0, "inserts": 212108, "deletes": 0, "compact_conflicts": 298, "split_conflicts": 39, "merge_conflicts": 0, "insert_conflicts": 2406, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 13, "memory_size": 118600860, "memory_size_index": 13577692, "allocated": 330778166664, "freed": 330659565804, "reclaimed": 330648420212, "reclaim_pending": 11145592, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577692, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1602, "num_rec_allocs": 4394252, "num_rec_frees": 4392650, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 13903255184, "lss_data_size": 400636, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 1669316, "est_recovery_size": 28140942, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 3620963, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 3408855, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.70192, "currSn": 35562, "gcSn": 35546, "gcSnIntervals": "[0 7609 18901 29933 35547]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 28, "num_trim_wctxs": 149, "num_readers": 12, "num_writers": 15, "page_bytes": 252818636, "page_cnt": 497, "page_itemcnt": 3857, "avg_item_size": 65548, "avg_page_size": 508689, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 131654104, "page_bytes_marshalled":8181281114, "page_bytes_compressed":4457447863, "compression_ratio": 1.83542, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 11712, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 504779241, "lss_data_size": 161855556, "lss_used_space": 700864416, "lss_disk_size": 737218560, "lss_fragmentation": 76, "lss_num_reads": 94489, "lss_read_bs": 43486035008, "lss_blk_read_bs": 44805562368, "bytes_written": 43955326976, "bytes_incoming": 37207011240, "write_amp": 2403.72, "write_amp_avg": 1.02, "lss_gc_num_reads": 94489, "lss_gc_reads_bs": 43486035008, "lss_blk_gc_reads_bs": 44805562368, "lss_gc_status": "frag 80, data: 119047096, used: 625656798 log:(37190870591 - 37816848384) head:37153197413", "lss_head_offset": 37415155316, "lss_tail_offset": 37966311424, "num_sctxs": 68, "num_free_sctxs": 53, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:36:55 [gsi-test] [] { "memory_quota": 1684757807, "count": 46854, "compacts": 56143, "purges": 0, "splits": 207, "merges": 0, "inserts": 46854, "deletes": 0, "compact_conflicts": 305, "split_conflicts": 30, "merge_conflicts": 0, "insert_conflicts": 4181, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 3, "memory_size": 125169132, "memory_size_index": 13577644, "allocated": 31725313400, "freed": 31600144268, "reclaimed": 31599619580, "reclaim_pending": 524688, "reclaim_list_size": 524688, "reclaim_list_count": 1, "reclaim_threshold": 1, "allocated_index": 13577644, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1702, "num_rec_allocs": 427633, "num_rec_frees": 425931, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 3071185992, "lss_data_size": 45459778, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 78378927, "est_recovery_size": 32269264, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 128543, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 81689, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 8.18269, "currSn": 2404, "gcSn": 2397, "gcSnIntervals": "[0 665 1308 1978 2398]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 4, "num_free_wctxs": 14, "num_trim_wctxs": 103, "num_readers": 4, "num_writers": 11, "page_bytes": 35527016, "page_cnt": 67, "page_itemcnt": 542, "avg_item_size": 65548, "avg_page_size": 530253, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 138614640, "page_bytes_marshalled":12982042332, "page_bytes_compressed":7072885218, "compression_ratio": 1.83547, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 24384, "num_total_smr": 8, "num_active_smr": 3, "num_pending_smr": 4, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 64454699, "lss_data_size": 61925118, "lss_used_space": 107961556, "lss_disk_size": 194564096, "lss_fragmentation": 42, "lss_num_reads": 383353, "lss_read_bs": 53050599149, "lss_blk_read_bs": 54601048064, "bytes_written": 53344784384, "bytes_incoming": 3071448184, "write_amp": 5.95, "write_amp_avg": 2.36, "lss_gc_num_reads": 383353, "lss_gc_reads_bs": 53050599149, "lss_blk_gc_reads_bs": 54601048064, "lss_gc_status": "frag 29, data: 47601540, used: 67714750, disk: 81611564, relocated: 914614, retries: 1248, skipped: 14391, cleaned:291431279, log:(7161037122 - 7228751872), disk:(7147140308-7228751872), activeFragRatio:30, activeMaxFragRatio:80, run:39 duration:20129 ms elapsed: 52196 ms", "lss_head_offset": 7161037122, "lss_tail_offset": 7236608000, "num_sctxs": 26, "num_free_sctxs": 15, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:36:56 [gsi-test] [] { "memory_quota": 1684757807, "count": 46598, "compacts": 3832, "purges": 413, "splits": 207, "merges": 0, "inserts": 46598, "deletes": 0, "compact_conflicts": 492, "split_conflicts": 49, "merge_conflicts": 0, "insert_conflicts": 6157, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 126473660, "memory_size_index": 13577836, "allocated": 15524365792, "freed": 15397892132, "reclaimed": 15396580852, "reclaim_pending": 1311280, "reclaim_list_size": 1311280, "reclaim_list_count": 2, "reclaim_threshold": 0, "allocated_index": 13577836, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 206, "total_records": 1722, "num_rec_allocs": 232732, "num_rec_frees": 231010, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 3054405704, "lss_data_size": 32821285, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 156291833, "est_recovery_size": 32403418, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 122763, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 76165, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 8.35922, "currSn": 2148, "gcSn": 2142, "gcSnIntervals": "[0 508 1170 1832 2143]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 14, "num_trim_wctxs": 96, "num_readers": 3, "num_writers": 12, "page_bytes": 22679608, "page_cnt": 43, "page_itemcnt": 346, "avg_item_size": 65548, "avg_page_size": 527432, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 138345120, "page_bytes_marshalled":4779602804, "page_bytes_compressed":2601746819, "compression_ratio": 1.83707, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 17792, "num_total_smr": 1, "num_active_smr": 3, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 504779241, "lss_data_size": 138581239, "lss_used_space": 663514726, "lss_disk_size": 762527744, "lss_fragmentation": 79, "lss_num_reads": 94653, "lss_read_bs": 43548125809, "lss_blk_read_bs": 44868440064, "bytes_written": 43980636160, "bytes_incoming": 37233623728, "write_amp": 2403.72, "write_amp_avg": 1.02, "lss_gc_num_reads": 94653, "lss_gc_reads_bs": 43548125809, "lss_blk_gc_reads_bs": 44868440064, "lss_gc_status": "frag 80, data: 119047096, used: 625656798 log:(37190870591 - 37816848384) head:37153197413", "lss_head_offset": 37461549323, "lss_tail_offset": 37987012608, "num_sctxs": 68, "num_free_sctxs": 55, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:37:00 [gsi-test] [] { "memory_quota": 1684757807, "count": 43489, "compacts": 52684, "purges": 0, "splits": 207, "merges": 0, "inserts": 43489, "deletes": 0, "compact_conflicts": 311, "split_conflicts": 32, "merge_conflicts": 0, "insert_conflicts": 3957, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 1, "memory_size": 124698784, "memory_size_index": 13577804, "allocated": 29577785068, "freed": 29453086284, "reclaimed": 29450070372, "reclaim_pending": 3015912, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577804, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1695, "num_rec_allocs": 398376, "num_rec_frees": 396681, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 2850616972, "lss_data_size": 7218494, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 32811336, "est_recovery_size": 27550720, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 120369, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 76880, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 8.14904, "currSn": 2220, "gcSn": 2218, "gcSnIntervals": "[0 504 1176 1854 2219]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 13, "num_trim_wctxs": 121, "num_readers": 4, "num_writers": 10, "page_bytes": 20844264, "page_cnt": 39, "page_itemcnt": 318, "avg_item_size": 65548, "avg_page_size": 534468, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 137229484, "page_bytes_marshalled":5514080058, "page_bytes_compressed":3004278356, "compression_ratio": 1.83541, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12928, "num_total_smr": 4, "num_active_smr": 3, "num_pending_smr": 1, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 504779241, "lss_data_size": 135106530, "lss_used_space": 623071537, "lss_disk_size": 750993408, "lss_fragmentation": 78, "lss_num_reads": 94799, "lss_read_bs": 43641645130, "lss_blk_read_bs": 44964872192, "bytes_written": 44036210688, "bytes_incoming": 37296287616, "write_amp": 2403.72, "write_amp_avg": 1.02, "lss_gc_num_reads": 94799, "lss_gc_reads_bs": 43641645130, "lss_blk_gc_reads_bs": 44964872192, "lss_gc_status": "frag 80, data: 119047096, used: 625656798 log:(37190870591 - 37816848384) head:37153197413", "lss_head_offset": 37550460347, "lss_tail_offset": 38032642048, "num_sctxs": 68, "num_free_sctxs": 52, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13T16:37:05.522-07:00 [Info] memstats {"Alloc":494370400, "TotalAlloc":3016776217424, "Sys":976251128, "Lookups":0, "Mallocs":180695756,"Frees":180665140, "HeapAlloc":494370400, "HeapSys":934182912, "HeapIdle":437035008, "HeapInuse":497147904,"HeapReleased":407003136, "HeapObjects":30616,"GCSys":31027200, "LastGC":1631576225522388213,"PauseTotalNs":129150044880, "PauseNs":[1810506, 4442863, 102563029, 5679389, 5097934, 1422058, 6716270, 1636681, 2754084, 909432, 183971413, 1443306, 308405885, 577564, 1785455, 5266584, 205333104, 76119694, 249839228, 1943382, 3840741, 5893438, 2993452, 5844543, 143051930, 1359718, 62368955, 6834565, 6138615, 141407914, 4834801, 2770393, 5517221, 3445437, 1279950, 2500326, 1450660, 1807837, 112458542, 1537844, 1238849, 217209053, 77320223, 4027211, 3754933, 1224177, 144929075, 56095381, 1287084, 1212026, 1475141, 1294679, 1480448, 4503421, 158568569, 6014309, 1204541, 5225599, 479956, 72356142, 146079500, 1950479, 1279515, 5660296, 84452871, 1549752, 1216818, 3337105, 163661348, 2815227, 1949369, 7159097, 1088428, 1076444, 148224423, 3574498, 4722127, 3998667, 161655091, 134464325, 4237485, 1845039, 1427026, 1307027, 227588978, 3531383, 4383675, 722678, 2097200, 5477972, 1431382, 5750962, 181176991, 4938010, 986918, 1250017, 5506596, 107716907, 2364507, 1463471, 1899994, 4451007, 1581628, 618003, 3165117, 4393160, 3884639, 177811785, 1434561, 1683927, 3174886, 1295720, 148375917, 1818597, 1888170, 2119609, 1684415, 128104606, 3090480, 218130697, 1623974, 3784008, 2360820, 1651326, 2203422, 51993579, 1057002, 2559560, 189391662, 2087167, 1431278, 1463403, 1332295, 854889, 1225549, 5274249], "NumGC":16463} Allocated: 2665459112, active: 2667184128, metadata: 32305760 (n_thp 0), resident: 2784591872, mapped: 2801958912, retained: 7962722304 2021-09-13 16:37:05 [gsi-test] [] Stop and verify... 2021-09-13 16:37:06 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper stopped 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:37:06 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : All daemons stopped 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:37:06 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper stopped 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:37:07 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : All daemons stopped 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:37:07 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper stopped 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:37:07 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : All daemons stopped 2021-09-13 16:37:07 [gsi-test] [0] Stop and verify... 2021-09-13 16:37:07 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:37:09 [gsi-test] [0] Stop and verify... OK 2021-09-13 16:37:09 [gsi-test] [1] Stop and verify... 2021-09-13 16:37:11 [gsi-test] [1] Stop and verify... OK 2021-09-13 16:37:11 [gsi-test] [2] Stop and verify... 2021-09-13 16:37:13 [gsi-test] [2] Stop and verify... OK 2021-09-13 16:37:13 [gsi-test] [3] Stop and verify... 2021-09-13 16:37:14 [gsi-test] [3] Stop and verify... OK 2021-09-13 16:37:14 [gsi-test] [4] Stop and verify... Mon Sep 13 16:37:14 PDT 2021 RSS = 3168664 2021-09-13 16:37:14 [gsi-test] [4] Stop and verify... OK 2021-09-13 16:37:14 [gsi-test] [5] Stop and verify... 2021-09-13 16:37:15 [gsi-test] [5] Stop and verify... OK 2021-09-13 16:37:15 [gsi-test] [6] Stop and verify... 2021-09-13 16:37:15 [gsi-test] [6] Stop and verify... OK 2021-09-13 16:37:15 [gsi-test] [7] Stop and verify... 2021-09-13 16:37:16 [gsi-test] [7] Stop and verify... OK 2021-09-13 16:37:16 [gsi-test] [8] Stop and verify... 2021-09-13 16:37:17 [gsi-test] [8] Stop and verify... OK 2021-09-13 16:37:17 [gsi-test] [9] Stop and verify... 2021-09-13 16:37:19 [gsi-test] [9] Stop and verify... OK 2021-09-13 16:37:19 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper started 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : all deamons started 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : all deamons started 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : all deamons started 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 16:37:19 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper started 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : all deamons started 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : all deamons started 2021-09-13 16:37:19 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper started 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : all deamons started 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : all deamons started 2021-09-13 16:37:19 [gsi-test] [] Stop and verify... OK 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:37:19 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 16:37:20 [plasma] [i] Plasma: mvccPurger: starting... purge_ratio 10.26923 2021-09-13 16:37:20 [ss-workload] instance inse% dele% read% 2021-09-13 16:37:20 [ss-workload] 0 14% 14% 70% 2021-09-13 16:37:20 [ss-workload] 1 14% 14% 70% 2021-09-13 16:37:20 [ss-workload] 2 14% 14% 70% 2021-09-13 16:37:20 [ss-workload] 3 14% 14% 70% 2021-09-13 16:37:20 [ss-workload] 4 3% 3% 92% 2021-09-13 16:37:20 [ss-workload] 5 3% 3% 92% 2021-09-13 16:37:20 [ss-workload] 6 3% 3% 92% 2021-09-13 16:37:20 [ss-workload] 7 18% 18% 62% 2021-09-13 16:37:20 [ss-workload] 8 18% 18% 62% 2021-09-13 16:37:28 [plasma] [i] Plasma: Adjusting SMO limits: Avg - Item Size 65548 Avg Page Size 24475977: Old MaxPageItems 400 MinPageItems 25 MaxDeltaChainLen 200 -> New MaxPageItems 4 MinPageItems 1 MaxDeltaChainLen 2 2021-09-13 16:37:31 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 23879 KB (expected = 10240 KB), wCtxCnt = 13, objCnt 1, changed reclaimList flush threshold from 50 to 0, changed reclaimSize flush threshold from 10240 KB to 630 KB. 2021-09-13 16:37:20 [ss-workload] 9 18% 18% 62% 2021-09-13 16:37:20 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:37:20 [ss-diag] 0 71150 69078 0 0 351 1 23007348 5 136114344 262416 1.000000 9.482332 1.000000 0:0:0 2021-09-13T16:37:35.852-07:00 [Info] memstats {"Alloc":500180560, "TotalAlloc":3056280979272, "Sys":976251128, "Lookups":0, "Mallocs":184014406,"Frees":183979566, "HeapAlloc":500180560, "HeapSys":934019072, "HeapIdle":431013888, "HeapInuse":503005184,"HeapReleased":364544000, "HeapObjects":34840,"GCSys":30994432, "LastGC":1631576255829720983,"PauseTotalNs":130243406803, "PauseNs":[49486188, 5149899, 1072448, 1156700, 1665669, 965747, 1352782, 1600299, 1150032, 1742811, 1813349, 2959177, 2093515, 1767981, 1495811, 1447017, 1498635, 2163877, 1637031, 1430446, 2157614, 851583, 1741346, 1545169, 1178938, 1207633, 988330, 1045978, 976628, 1195624, 997266, 899913, 1535103, 896684, 668863, 1325943, 1221741, 637609, 1572104, 1448772, 823473, 1202830, 785328, 1126881, 1443078, 1527279, 746600, 1946991, 1495464, 1618990, 1561838, 987808, 995728, 410969, 1154716, 1524948, 1634655, 1485253, 1504378, 929803, 1646587, 1364837, 779943, 826890, 1620381, 1343544, 1397330, 755539, 1432342, 1663139, 1550690, 1739039, 1495749, 2015062, 962588, 1012846, 1382651, 1043598, 998049, 1325674, 1626886, 1180306, 649333, 1187801, 1331506, 1587119, 2100547, 1672268, 1407900, 1314463, 1494554, 1249815, 1345379, 456627, 783495, 1536502, 1800664, 1247455, 1228974, 1568244, 1550906, 1310100, 100660130, 6171105, 1663459, 8305723, 6181548, 1448116, 1353213, 2688726, 89639971, 4630172, 960310, 1088645, 90664341, 1783865, 1746570, 1440570, 1363497, 3211539, 2527108, 3636524, 3962329, 1074165, 1616718, 1568452, 210288137, 1451025, 1856375, 1463902, 3697662, 4370400, 1281614, 1118948, 853153, 2897253, 5269087, 6278926, 1050521, 2947836, 356104, 3638943, 8177401, 14337750, 13530113, 18828830, 19008089, 22590698, 15216081, 26380673, 34083815, 15106787, 2443769, 12720647, 17134527, 24410626, 16456306, 28994469, 27098073], "NumGC":16622} Allocated: 6406143680, active: 6407864320, metadata: 32305760 (n_thp 0), resident: 6518718464, mapped: 6546571264, retained: 4218109952 2021-09-13 16:37:39 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: starting... frag 98, data: 3327872, used: 183344598 log:(5083685028 - 5267423232) head:5069926646 2021-09-13 16:37:39 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: completed... frag 89, data: 13687336, used: 134106644, disk: 138845798, relocated: 0, retries: 0, skipped: 11618227, cleaned:860631040, log:(5944316068 - 6078947328), disk:(5940101530-6078947328), activeFragRatio:30, activeMaxFragRatio:80, run:288 duration:2515 ms elapsed: 299447 ms 2021-09-13 16:37:50 [plasma] [i] Plasma: mvccPurger: finished... purge_ratio 7.22927 2021-09-13 16:37:20 [ss-diag] 1 241195 239707 0 0 18784 51 1231253632 17 98597880 1049472 1.000000 14.986459 1.000000 0:0:0 2021-09-13 16:37:20 [ss-diag] 2 70198 68426 0 0 337 1 22089676 4 116371572 196780 1.000000 0.938865 1.000000 0:0:0 2021-09-13 16:37:20 [ss-diag] 3 67576 65433 0 0 358 1 23466184 5 140770964 262368 1.000000 0.938865 1.000000 0:0:0 2021-09-13 16:37:20 [ss-diag] 4 4735218 4733664 0 0 3263 424 213883124 208 115456604 13577836 1.000000 2.549321 1.000000 0:0:0 2021-09-13 16:37:20 [ss-diag] 5 4633022 4631442 0 0 3743 475 245346164 208 117159716 13578028 1.000000 0.938865 1.000000 0:0:0 2021-09-13 16:37:20 [ss-diag] 6 4642415 4640867 0 0 4150 521 272024200 208 115060836 13577692 1.000000 0.938865 1.000000 0:0:0 2021-09-13 16:37:20 [ss-diag] 7 450334 448618 0 0 216 27 14158368 208 126085892 13577644 1.000000 4.912591 1.000000 0:0:0 2021-09-13 16:37:20 [ss-diag] 8 251461 249769 0 0 226 28 14813848 208 124507556 13577836 1.000000 0.938865 1.000000 0:0:0 2021-09-13 16:37:20 [ss-diag] 9 420729 419027 0 0 294 37 19271112 208 125157676 13577804 1.000000 0.938865 1.000000 0:0:0 2021-09-13T16:38:06.203-07:00 [Info] memstats {"Alloc":480441816, "TotalAlloc":3086394167896, "Sys":976283896, "Lookups":0, "Mallocs":186726485,"Frees":186697878, "HeapAlloc":480441816, "HeapSys":933986304, "HeapIdle":450756608, "HeapInuse":483229696,"HeapReleased":34791424, "HeapObjects":28607,"GCSys":31129600, "LastGC":1631576286202466948,"PauseTotalNs":132175752353, "PauseNs":[31655693, 37893219, 4001760, 2431223, 1988961, 500001, 543087, 1102354, 1582560, 1231191, 2202703, 1125200, 666332, 3268884, 2136159, 2289510, 679743, 3839890, 1247465, 2541150, 4912178, 4690239, 2664126, 1387111, 375046, 4750322, 2921196, 774213, 1898331, 808730, 1363656, 1273473, 1061091, 2680198, 883590, 2588304, 1907947, 26444236, 1561993, 1013574, 1319564, 857819, 68038146, 2153519, 751857, 84588949, 1657235, 974069, 3983565, 830132, 2423885, 757943, 4993719, 1075182, 1795799, 4972490, 1655617, 928457, 1473303, 2276290, 644330, 1522806, 5446848, 3894979, 5094921, 1308306, 127759744, 73878946, 4618806, 122805605, 1425705, 961113, 2318018, 2835852, 31032204, 97092919, 738899, 1294425, 2778540, 1523306, 1619324, 75904279, 2082517, 29371707, 152649304, 1451986, 103918800, 26728747, 822329, 2258596, 10710899, 2182479, 1530760, 1604911, 155599053, 2810615, 1020447, 1191083, 1378297, 750435, 47603651, 10479712, 5974140, 4908553, 231199817, 1853257, 1010630, 3283584, 10005516, 14828637, 2832478, 3732327, 6184727, 1581657, 19790140, 1918209, 4050552, 2376065, 136797085, 977794], "NumGC":16742} Allocated: 3164854144, active: 3166543872, metadata: 36533904 (n_thp 0), resident: 3393871872, mapped: 3421712384, retained: 9438023680 Mon Sep 13 16:38:14 PDT 2021 RSS = 3629960 2021-09-13 16:38:20 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoveryCleaner: starting... frag 97, data: 401688, used: 17354356 log:(11849984 - 29466624) head:10793158 2021-09-13 16:38:20 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoveryCleaner: completed... frag 35, data: 17186584, used: 26795612, disk: 28901096, relocated: 16786, retries: 0, skipped: 12647, cleaned:376557732, log:(388407716 - 415203328), disk:(386302232-415203328), activeFragRatio:30, activeMaxFragRatio:80, run:46 duration:12382 ms elapsed: 293245 ms 2021-09-13 16:38:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 96, data: 532820, used: 17342172 log:(16535844 - 33878016) head:14291410 2021-09-13 16:38:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 97, data: 401688, used: 16551644, disk: 18001698, relocated: 0, retries: 0, skipped: 1591, cleaned:9818344, log:(26354188 - 43171840), disk:(25170142-43171840), activeFragRatio:30, activeMaxFragRatio:80, run:12 duration:491 ms elapsed: 298489 ms 2021-09-13 16:38:25 [plasma] [i] Plasma: mvccPurger: starting... purge_ratio 10.33654 2021-09-13 16:38:33 [plasma] [i] Plasma: Adjusting SMO limits: Avg - Item Size 65548 Avg Page Size 23824176: Old MaxPageItems 400 MinPageItems 25 MaxDeltaChainLen 200 -> New MaxPageItems 4 MinPageItems 1 MaxDeltaChainLen 2 2021-09-13 16:38:33 [plasma] [i] Plasma: mvccPurger: finished... purge_ratio 7.00000 2021-09-13T16:38:36.610-07:00 [Info] memstats {"Alloc":445855728, "TotalAlloc":3123836239560, "Sys":976283896, "Lookups":0, "Mallocs":190159467,"Frees":190125081, "HeapAlloc":445855728, "HeapSys":933462016, "HeapIdle":484810752, "HeapInuse":448651264,"HeapReleased":206618624, "HeapObjects":34386,"GCSys":30965760, "LastGC":1631576316507104052,"PauseTotalNs":134060716551, "PauseNs":[1723174, 2643986, 3492632, 5263468, 4843451, 2314433, 1101685, 3855334, 1660493, 366069, 6165468, 48433294, 16908980, 1638541, 1218424, 1264055, 1724555, 1409796, 955124, 1447351, 77229296, 7085309, 1788735, 2132759, 6590479, 899205, 860914, 1198375, 1233732, 1550343, 5796347, 3433723, 155338845, 828008, 860557, 1104319, 17660592, 1337620, 3955977, 4904401, 2165201, 1410938, 3399559, 2810132, 3613182, 1309971, 7052275, 930806, 8069743, 12165419, 2027319, 1389297, 1509946, 1739377, 3556674, 2600314, 6134788, 4532293, 3898688, 867088, 4927278, 4544746, 4972381, 3264242, 4699152, 3400104, 1646344, 3073044, 1082600, 1373046, 1642596, 2441361, 4123028, 4607141, 4823134, 1580291, 6594954, 7025413, 103773737, 3521678, 186467149, 106360448, 3087472, 749918, 3102426, 2095592, 174365516, 2681694, 7790203, 1904216, 683046, 2676782, 1383688, 1158630, 2970117, 1736669, 1604081, 2781025, 1647321, 1002200, 2669805, 3088899, 3156656, 1933168, 5049013, 986110, 219353895, 946588, 1984988, 3715954, 5603771, 4545251, 5022056, 2287403, 2296429, 1249633, 2989959, 1653827, 1183150, 153585480, 653482, 1577687, 5611931, 2029666, 9116655, 2545220, 3463259, 126918151, 846564, 795841, 1141295, 1735453, 4442064, 1640241, 1162461, 1764252, 765051, 754460, 1307512, 804532, 9001266, 11418611, 9074925, 14704627, 16546742, 19713517, 14236456, 12625062, 5999179, 12125642, 5131479, 7241558], "NumGC":16894} Allocated: 3469052392, active: 3470675968, metadata: 36533904 (n_thp 0), resident: 3725688832, mapped: 3760869376, retained: 9098866688 2021-09-13 16:38:58 [ss-workload] instance inse% dele% read% 2021-09-13 16:38:58 [ss-workload] 0 14% 14% 70% 2021-09-13 16:38:58 [ss-workload] 1 14% 14% 70% 2021-09-13 16:38:58 [ss-workload] 2 14% 14% 70% 2021-09-13 16:38:58 [ss-workload] 3 14% 14% 70% 2021-09-13 16:38:58 [ss-workload] 4 3% 3% 92% 2021-09-13 16:38:58 [ss-workload] 5 3% 3% 92% 2021-09-13 16:38:58 [ss-workload] 6 3% 3% 92% 2021-09-13T16:39:06.993-07:00 [Info] memstats {"Alloc":405575224, "TotalAlloc":3159699141152, "Sys":976283896, "Lookups":0, "Mallocs":193341220,"Frees":193307841, "HeapAlloc":405575224, "HeapSys":934150144, "HeapIdle":525910016, "HeapInuse":408240128,"HeapReleased":377085952, "HeapObjects":33379,"GCSys":30998528, "LastGC":1631576346978071851,"PauseTotalNs":136080347609, "PauseNs":[7205505, 12912073, 18299430, 6705019, 15218425, 13380729, 219670224, 7180507, 18668325, 12489357, 14538939, 24883629, 17529571, 173671814, 18212833, 5656246, 1961979, 1853505, 1006358, 3369221, 1949006, 3112497, 3922519, 1551802, 9496230, 6200046, 9378131, 173866862, 4324500, 1516664, 963881, 14645462, 4162528, 2060209, 5056746, 1694571, 8928259, 7654416, 9950398, 8483332, 11002639, 23270117, 8012149, 15675015, 5132334, 27858561, 1524721, 6425951, 9341303, 10017781, 6971302, 14911174, 16920226, 15074717, 17997632, 11456610, 22431275, 21940249, 15075751, 11841162, 1238605, 4889565, 1145935, 5803966, 2684195, 11338664, 843696, 3801665, 3056720, 13443300, 5417566, 2791484, 1416599, 1598428, 1379212, 1539331, 1216333, 3800801, 581732, 1847282, 1376966, 1668699, 2946859, 1378791, 4427457, 88768423, 4273768, 7438513, 9696009, 9859905, 11359154, 10081429, 7893767, 540817, 13637815, 6050483, 29309561, 16949746, 21287759, 14355810, 3466524, 3739784, 3316541, 2590991, 7169105, 15811524, 12808854, 14964498, 19772665, 16423625, 19149068, 20980619, 19752871, 26497678, 19453802, 13593426, 28411051, 7316484, 14733886, 3423939, 4223130, 1034944, 1859583, 96606456, 2162835, 1346670, 960856, 1678462, 127970343, 1391704, 5244829, 1090906, 1454212, 588295, 1181849, 1066154, 1006575, 935858, 2003545, 1426360, 3800673, 5887208, 891938, 1091511], "NumGC":17038} Allocated: 3824199040, active: 3825807360, metadata: 36556816 (n_thp 0), resident: 4198449152, mapped: 4226265088, retained: 8901906432 2021-09-13 16:38:58 [ss-workload] 7 18% 18% 62% 2021-09-13 16:38:58 [ss-workload] 8 18% 18% 62% 2021-09-13 16:38:58 [ss-workload] 9 18% 18% 62% 2021-09-13 16:38:58 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:38:58 [ss-diag] 0 74214 72128 0 0 207 1 13568436 6 137097124 328004 1.000000 12.804965 1.000000 0:0:0 2021-09-13 16:38:58 [ss-diag] 1 261126 259374 0 0 481 58 31528588 206 128317644 13446564 1.000000 106.570638 1.000000 0:0:0 2021-09-13 16:38:58 [ss-diag] 2 72562 70704 0 0 335 1 21958580 4 122012460 196780 1.000000 1.007343 1.000000 0:0:0 2021-09-13 16:38:58 [ss-diag] 3 132180 130570 0 0 398 30 26088104 166 116375252 10822708 1.000000 1.007343 1.000000 0:0:0 2021-09-13 16:38:58 [ss-diag] 4 5071221 5069593 0 0 5316 674 348453168 208 120307492 13577836 1.000000 2.606633 1.000000 0:0:0 2021-09-13 16:38:58 [ss-diag] 5 4998494 4996852 0 0 4879 618 319808692 208 121223228 13578028 1.000000 1.007343 1.000000 0:0:0 2021-09-13 16:38:58 [ss-diag] 6 4983787 4982191 0 0 4948 634 324331504 208 118207588 13577692 1.000000 1.007343 1.000000 0:0:0 2021-09-13 16:38:58 [ss-diag] 7 477114 475440 0 0 181 23 11864188 208 123333244 13577644 1.000000 4.512443 1.000000 0:0:0 2021-09-13 16:38:58 [ss-diag] 8 282977 281287 0 0 498 62 32642904 208 124378428 13577836 1.000000 1.007343 1.000000 0:0:0 2021-09-13 16:38:58 [ss-diag] 9 452996 451328 0 0 445 54 29168860 208 122929348 13577804 1.000000 1.007343 1.000000 0:0:0 Mon Sep 13 16:39:14 PDT 2021 RSS = 4108852 2021-09-13 16:39:19 [gsi-test] [] QuotaTuner: 1684757807 -> 1516282026 2021-09-13T16:39:37.350-07:00 [Info] memstats {"Alloc":358370880, "TotalAlloc":3203056853976, "Sys":976283896, "Lookups":0, "Mallocs":197220225,"Frees":197193555, "HeapAlloc":358370880, "HeapSys":933953536, "HeapIdle":572514304, "HeapInuse":361439232,"HeapReleased":344514560, "HeapObjects":26670,"GCSys":30900224, "LastGC":1631576377201343514,"PauseTotalNs":137535513892, "PauseNs":[1228918, 20721176, 4727526, 900190, 1914257, 177940400, 1481827, 6040072, 5870513, 1646896, 1404130, 2265888, 4872942, 1550936, 954399, 4253886, 3215531, 7730192, 3941592, 1210112, 1104784, 829047, 5039294, 857330, 811581, 698024, 1625807, 1307934, 12781961, 3327532, 810997, 2214897, 2423884, 1319139, 1797891, 17363663, 1838536, 2146544, 130084165, 529650, 1545515, 4936362, 38376367, 1644575, 4404844, 3082723, 4776186, 4763420, 4178195, 948489, 926867, 1487230, 870427, 742713, 5062587, 3827155, 1737962, 184226265, 1152380, 1257645, 1376416, 1326537, 1526854, 1642713, 613012, 5869304, 91950730, 57050928, 850906, 1465173, 4374846, 5269675, 73254312, 5770097, 17551665, 1824646, 2321165, 1150423, 1832873, 17241201, 15452023, 2991240, 732322, 3508174, 2303064, 2341598, 3856055, 1258016, 4461267, 2045960, 759043, 1215048, 25051230, 23293930, 4129457, 691984, 13060802, 2636615, 4321649, 2429729, 1777249, 524441, 4763935, 5681109, 5241592, 1761765, 9083292, 1707149, 1432203, 1472794, 6516995, 686953, 3142299, 717310, 1133448, 1534746, 2634937, 859618, 1710604, 1157098, 1277194, 4580757, 1255955, 4126712, 5626425, 2667497, 1231650, 1015463, 3559992, 3879744, 1699334, 2279671, 1326785, 816169, 1239189, 59022671, 41901969, 3525894, 14263451, 8132971, 1161582, 1292944, 856336, 2925823, 11385443, 2608682, 1028738, 983461, 812082, 6458718, 2766452, 2375137, 3061978, 1171912, 1568541, 1024856, 517967, 2579106, 1978785, 675451, 1919010, 2340941, 4910331, 32024306, 866440, 13366313, 1045016, 1150507, 1325598, 2489030, 1409803, 1460267, 492266, 1538796, 1090003], "NumGC":17213} Allocated: 4106691408, active: 4108259328, metadata: 36591248 (n_thp 0), resident: 4308488192, mapped: 4336271360, retained: 9127444480 2021-09-13T16:40:07.897-07:00 [Info] memstats {"Alloc":476692320, "TotalAlloc":3241577889088, "Sys":976283896, "Lookups":0, "Mallocs":200565909,"Frees":200531661, "HeapAlloc":476692320, "HeapSys":934084608, "HeapIdle":454729728, "HeapInuse":479354880,"HeapReleased":396304384, "HeapObjects":34248,"GCSys":30900224, "LastGC":1631576407762175634,"PauseTotalNs":139988377606, "PauseNs":[560472, 1011587, 1015849, 887585, 1786325, 1492924, 1793570, 1509872, 2121040, 2001497, 1669551, 2237844, 3086172, 645087, 1377229, 879836, 4154953, 28731463, 1318371, 1229547, 1279337, 2432015, 383183, 1610114, 2898451, 1539562, 1375232, 1758430, 1484752, 1897562, 7704787, 185696441, 15438522, 13928365, 2725156, 1265635, 12804722, 3919276, 14562678, 13856252, 2348617, 25554416, 2755516, 9212687, 13889231, 8727411, 19349108, 162431237, 1583458, 7700663, 6651018, 14631280, 1564490, 51751450, 11103684, 28722641, 3984830, 125790044, 14453725, 5772266, 147765249, 19405516, 22690530, 19553572, 3625005, 9797730, 500833, 1078645, 1612944, 1798473, 452572, 1154016, 988797, 1129916, 541776, 1109908, 1539015, 238292862, 4842033, 1879943, 3746104, 1534025, 349807, 4750014, 4258056, 5348566, 127490753, 1685673, 2104830, 1535940, 1171659, 28827702, 2463612, 1513342, 1594831, 448496, 4530010, 2020309, 3582215, 210772821, 787168, 2177879, 21961157, 8320297, 679658, 1671829, 966256, 1479028, 1364846, 786048, 2577853, 1302058, 1554380, 1007303, 1620977, 1525979, 4688538, 2106121, 1656290, 4095341, 1147768, 1552105, 1678739, 6086318, 1499481, 1468576, 1375779, 1514568, 1352403, 685145, 99790566, 1172082, 702125, 862510, 188070675, 2254573, 5648525, 1798999, 1637197, 779808, 1426878, 3154135, 1974858, 773514, 69524886, 3928680, 947645, 1399039, 62046982, 1331910, 2035401, 155102048, 3420091, 4760078, 2093503], "NumGC":17368} Allocated: 4014136552, active: 4015661056, metadata: 38800784 (n_thp 0), resident: 4257030144, mapped: 4282605568, retained: 9852198912 2021-09-13 16:40:14 [ss-workload] instance inse% dele% read% 2021-09-13 16:40:14 [ss-workload] 0 14% 14% 70% Mon Sep 13 16:40:15 PDT 2021 RSS = 4567840 2021-09-13 16:40:14 [ss-workload] 1 14% 14% 70% 2021-09-13 16:40:14 [ss-workload] 2 14% 14% 70% 2021-09-13 16:40:14 [ss-workload] 3 14% 14% 70% 2021-09-13 16:40:14 [ss-workload] 4 3% 3% 92% 2021-09-13 16:40:14 [ss-workload] 5 3% 3% 92% 2021-09-13 16:40:14 [ss-workload] 6 3% 3% 92% 2021-09-13 16:40:19 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Enable page eviction before reaching quota. 2021-09-13 16:40:19 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Enable page eviction before reaching quota. 2021-09-13 16:40:19 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Enable page eviction before reaching quota. 2021-09-13 16:40:14 [ss-workload] 7 18% 18% 62% 2021-09-13 16:40:14 [ss-workload] 8 18% 18% 62% 2021-09-13 16:40:14 [ss-workload] 9 18% 18% 62% 2021-09-13 16:40:14 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:40:14 [ss-diag] 0 75955 73643 0 0 358 1 23466184 6 151921228 328004 1.000000 13.920731 1.000000 0:0:0 2021-09-13 16:40:14 [ss-diag] 1 277565 275795 0 0 268 31 17566864 208 129626748 13577788 1.000000 3.855729 1.000000 0:0:0 2021-09-13 16:40:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 158, "num_active_sctxs": 43, "num_free_sctxs": 114, "num_trim_sctxs": 9, "num_wctxs": 214, "num_free_wctxs": 196, "num_active_wctxs": 18, "num_trim_wctxs": 1277, "num_readers": 72, "num_writers": 107, "cpu": 9.94992, "total_core": 16, "threads": 55, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3393015808, "memory_jemalloc_allocated": 2872155560, "memory_in_use": 3393015808, "memory_stats_size_total": 2151871721, "memory_stats_size_global": 1382024976, "memory_stats_size_page": 1272927212, "memory_stats_size_index": 109212580, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 181000, "reclaim_pending": 10555508, "buf_memused": 759176421, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 17, "num_active_smr": 8, "num_pending_smr": 4, "num_pages": 1675, "items_count": 2080, "total_records": 17750, "total_ops": 961010, "compacts": 2477677, "purges": 1063, "lss_data_size": 238935280, "lss_used_space": 1135067446, "lss_fragmentation": 78, "lss_num_reads": 733988, "lss_read_bs": 139464794568, "lss_blk_read_bs": 142461091840, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 140473835520, "bytes_incoming": 49798519848, "write_amp_avg": 1.52, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 17750, "resident_ratio": 100.00000, "cleaner_num_reads": 733988, "cleaner_read_bs": 139464794568, "cleaner_blk_read_bs": 142461091840, "recovery_num_lssctx": 5, "recovery_num_sctxs": 29, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 9, "recovery_num_trim_sctxs": 9, "recovery_buf_memused": 5242880, "recovery_data_size": 29259556, "recovery_used_space": 257206580, "recovery_lss_blk_read_bs": 81016782848, "recovery_lss_read_bs": 78706116512, "recovery_lss__num_reads": 611359, "recovery_bytes_written": 79136378880, "recovery_lss_frag": 88, "recovery_cleaner_num_reads": 611359, "recovery_cleaner_read_bs": 78706116512, "recovery_cleaner_blk_read_bs": 81016782848, "mvcc_purge_ratio": 8.53365 }2021-09-13 16:40:22 [plasma] [i] Plasma: mvccPurger: starting... purge_ratio 11.20192 2021-09-13 16:40:14 [ss-diag] 2 75331 73383 0 0 340 1 22286320 5 127980312 262368 1.000000 0.594278 1.000000 0:0:0 2021-09-13 16:40:14 [ss-diag] 3 151974 150190 0 0 238 29 15600424 208 130536020 13577580 1.000000 0.594278 1.000000 0:0:0 2021-09-13 16:40:14 [ss-diag] 4 5407646 5405998 0 0 5494 700 360120712 208 121617396 13577836 1.000000 1.753399 1.000000 0:0:0 2021-09-13 16:40:14 [ss-diag] 5 5347344 5345728 0 0 5095 657 333967060 208 119519028 13578028 1.000000 0.594278 1.000000 0:0:0 2021-09-13 16:40:14 [ss-diag] 6 5344938 5343298 0 0 5264 674 345044672 208 121091924 13577692 1.000000 0.594278 1.000000 0:0:0 2021-09-13 16:40:14 [ss-diag] 7 506512 504802 0 0 616 76 40377568 208 125690508 13577644 1.000000 3.810817 1.000000 0:0:0 2021-09-13 16:40:14 [ss-diag] 8 311988 310261 0 0 443 54 29037764 208 126799264 13577836 1.000000 0.594278 1.000000 0:0:0 2021-09-13 16:40:25 [plasma] [i] Plasma: Adjusting SMO limits: Avg - Item Size 65548 Avg Page Size 16073059: Old MaxPageItems 400 MinPageItems 25 MaxDeltaChainLen 200 -> New MaxPageItems 4 MinPageItems 1 MaxDeltaChainLen 2 2021-09-13 16:40:26 [plasma] [i] Plasma: mvccPurger: finished... purge_ratio 7.00000 2021-09-13 16:40:14 [ss-diag] 9 481969 480279 0 0 306 38 20057688 208 124370428 13577804 1.000000 0.594278 1.000000 0:0:0 2021-09-13 16:40:30 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 26825 KB (expected = 10240 KB), wCtxCnt = 14, objCnt 7, changed reclaimList flush threshold from 50 to 0, changed reclaimSize flush threshold from 10240 KB to 585 KB. 2021-09-13 16:40:36 [gsi-test] [9] CreateRecoveryPoint() - DO!!! 2021-09-13T16:40:38.178-07:00 [Info] memstats {"Alloc":381225024, "TotalAlloc":3278461973920, "Sys":976283896, "Lookups":0, "Mallocs":203773434,"Frees":203746265, "HeapAlloc":381225024, "HeapSys":932118528, "HeapIdle":547774464, "HeapInuse":384344064,"HeapReleased":371826688, "HeapObjects":27169,"GCSys":30900224, "LastGC":1631576438083640586,"PauseTotalNs":141718907036, "PauseNs":[808599, 1565002, 1392131, 563905, 1089927, 846978, 1262176, 775117, 1003189, 710824, 1065839, 1191143, 1419698, 1752902, 1258503, 2536532, 7658771, 808882, 1545172, 2310545, 808844, 886812, 113558769, 54915145, 1745521, 6445909, 3026809, 4857656, 5364096, 1934895, 48198903, 1272582, 4839930, 1581723, 1010219, 1025453, 735308, 1550069, 1423945, 1786010, 3629662, 5539515, 485233, 530064, 4329157, 1734864, 842941, 2495042, 1014881, 19166327, 1189362, 924472, 1628591, 964778, 927300, 206169013, 1036054, 4182726, 2031226, 1695346, 8974847, 1266529, 1043342, 2175274, 3031105, 188475465, 1876569, 1145773, 3388126, 1714169, 1258785, 1679515, 1030959, 1640158, 2603146, 571760, 1272333, 5773681, 5272463, 1242271, 1597884, 1812299, 1169086, 3624671, 1522903, 1717675, 1663162, 1190808, 3631739, 5168345, 1541853, 1345866, 6618159, 2029814, 1124385, 2919048, 2160621, 1192987, 1333382, 1255111, 1673663, 1221844, 1755598, 927714, 5458881, 10231735, 20041550, 19085652, 198872933, 25351203, 28277927, 28900093, 2766824, 61610914, 121988718, 13536819, 18737405, 20490253, 25017668, 21469592, 24816432, 33656598, 12438050, 900613, 2009528, 758458, 1058833, 13538037, 4083861, 374651, 1734571, 629569, 2360625, 1680781, 1574441, 1642244, 135660219, 21491507, 1300297, 1785204, 1552649, 1347760, 749192, 2303387, 415027, 747841, 1293758, 1463274, 637982], "NumGC":17517} Allocated: 3515687016, active: 3517206528, metadata: 38800784 (n_thp 0), resident: 4095381504, mapped: 4120956928, retained: 10013847552 2021-09-13 16:40:39 [gsi-test] [6] CreateRecoveryPoint() - DO!!! 2021-09-13 16:40:42 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 16:40:43 [gsi-test] [5] CreateRecoveryPoint() - DO!!! 2021-09-13 16:40:43 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 16:40:52 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 16:40:57 [gsi-test] [8] CreateRecoveryPoint() - DO!!! 2021-09-13 16:40:59 [gsi-test] [3] CreateRecoveryPoint() - DO!!! 2021-09-13 16:41:01 [gsi-test] [2] CreateRecoveryPoint() - DO!!! 2021-09-13 16:41:04 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: starting... frag 41, data: 44210331, used: 75452546 log:(6869605843 - 6945771520) head:6865962237 2021-09-13 16:41:04 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: completed... frag 30, data: 46178855, used: 65970550, disk: 104252244, relocated: 4571539, retries: 6785, skipped: 68051, cleaned:1488543927, log:(8358149770 - 8424120320), disk:(8319868076-8424120320), activeFragRatio:30, activeMaxFragRatio:80, run:183 duration:130759 ms elapsed: 301660 ms 2021-09-13 16:41:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 90, data: 3942216, used: 39894550 log:(51672613074 - 51713032192) head:51671560332 2021-09-13 16:41:06 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 96, data: 3089876, used: 88477206, disk: 89529948, relocated: 50565385, retries: 337353, skipped: 12204081, cleaned:6061461784, log:(57734074858 - 57822552064), disk:(57733022116-57822552064), activeFragRatio:30, activeMaxFragRatio:80, run:152 duration:288737 ms elapsed: 302396 ms 2021-09-13T16:41:08.365-07:00 [Info] memstats {"Alloc":437134928, "TotalAlloc":3306719915312, "Sys":976283896, "Lookups":0, "Mallocs":206264223,"Frees":206230023, "HeapAlloc":437134928, "HeapSys":933658624, "HeapIdle":493723648, "HeapInuse":439934976,"HeapReleased":367771648, "HeapObjects":34200,"GCSys":30998528, "LastGC":1631576468211325383,"PauseTotalNs":142618287769, "PauseNs":[1335766, 1980775, 1477511, 1009492, 43876159, 1715181, 598496, 21639761, 1171073, 1769669, 735123, 1284872, 3368399, 755682, 1015800, 636949, 3199945, 3533972, 8128599, 526318, 7385771, 1450850, 1420245, 7971076, 952613, 1091282, 2368940, 58085493, 5274002, 24788172, 5337816, 3278662, 2510410, 64922739, 10394402, 4163631, 1129948, 4028748, 958271, 4513109, 2677989, 3602732, 1658645, 10243327, 1296145, 851611, 24600333, 7025910, 7295353, 11866314, 2224875, 18776118, 1812507, 839161, 1173353, 1356689, 2718460, 2898889, 3495386, 10879625, 1031602, 982717, 3934687, 75006956, 1608444, 433587, 486119, 28402148, 562762, 1320671, 1364582, 920880, 1100807, 5520282, 5936215, 2286431, 1904106, 1599935, 1570046, 1339001, 1771451, 3739609, 8705576, 3052923, 5825270, 717545, 8851424, 9897243, 9434884, 1198427, 4817117, 14372902, 4946731, 12510080, 7623770, 8912931, 13207442, 17109079, 17347451, 13213886, 974114, 8187626, 19745621, 3449584, 5081137, 9893080, 17971534, 11123845, 27313960, 5696353, 26966136, 15366597, 13964283], "NumGC":17630} Allocated: 3315898800, active: 3317338112, metadata: 45751184 (n_thp 0), resident: 3606499328, mapped: 3625127936, retained: 13932228608 2021-09-13 16:41:13 [gsi-test] [1] CreateRecoveryPoint() - DO!!! Mon Sep 13 16:41:15 PDT 2021 RSS = 3894244 2021-09-13 16:41:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 45, data: 16334208, used: 30023231 log:(44952330380 - 44982353920) head:44951277638 2021-09-13 16:41:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 60, data: 15154020, used: 38121244, disk: 39174004, relocated: 60397086, retries: 28614, skipped: 4502398, cleaned:5814820934, log:(50767151314 - 50805272576), disk:(50766098572-50805272576), activeFragRatio:30, activeMaxFragRatio:80, run:178 duration:284575 ms elapsed: 302230 ms 2021-09-13 16:41:19 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Disable page eviction before reaching quota. 2021-09-13 16:41:19 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Disable page eviction before reaching quota. 2021-09-13 16:41:19 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Disable page eviction before reaching quota. 2021-09-13 16:41:24 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: starting... frag 88, data: 7754304, used: 69185301 log:(23439316595 - 23508824064) head:23432925192 2021-09-13 16:41:24 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: completed... frag 92, data: 9577334, used: 131641292, disk: 160456549, relocated: 3797724, retries: 97883, skipped: 23513, cleaned:3257014721, log:(26696331316 - 26827972608), disk:(26667516059-26827972608), activeFragRatio:30, activeMaxFragRatio:80, run:47 duration:286652 ms elapsed: 300310 ms 2021-09-13 16:41:26 [plasma] [i] Plasma: mvccPurger: starting... purge_ratio 10.13462 2021-09-13 16:41:27 [ss-workload] instance inse% dele% read% 2021-09-13 16:41:27 [ss-workload] 0 14% 14% 70% 2021-09-13 16:41:27 [ss-workload] 1 14% 14% 70% 2021-09-13 16:41:34 [plasma] [i] Plasma: Adjusting SMO limits: Avg - Item Size 65548 Avg Page Size 26809132: Old MaxPageItems 400 MinPageItems 25 MaxDeltaChainLen 200 -> New MaxPageItems 4 MinPageItems 1 MaxDeltaChainLen 2 2021-09-13 16:41:34 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: starting... frag 31, data: 66936690, used: 97763818 log:(4765605380 - 4863369216) head:4750657104 2021-09-13 16:41:34 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: completed... frag 29, data: 44998205, used: 64185269, disk: 87008632, relocated: 20469, retries: 9066, skipped: 5312, cleaned:2612885389, log:(7378490769 - 7443640320), disk:(7356631688-7443640320), activeFragRatio:30, activeMaxFragRatio:80, run:49 duration:146629 ms elapsed: 300618 ms 2021-09-13T16:41:38.607-07:00 [Info] memstats {"Alloc":390256056, "TotalAlloc":3341947355128, "Sys":976283896, "Lookups":0, "Mallocs":209453090,"Frees":209419816, "HeapAlloc":390256056, "HeapSys":934215680, "HeapIdle":540934144, "HeapInuse":393281536,"HeapReleased":390414336, "HeapObjects":33274,"GCSys":30900224, "LastGC":1631576498486405331,"PauseTotalNs":144752598590, "PauseNs":[26135413, 9683365, 24102364, 866321, 20681559, 24315251, 4406515, 842216, 6367315, 3898264, 16650119, 10197327, 24809261, 2833246, 27958577, 11582223, 175940045, 15154875, 17058254, 56771212, 7049056, 40192795, 1007185, 59627199, 60615039, 1114706, 11346010, 27752786, 13707519, 712776, 176937096, 21273153, 15087478, 63855528, 1083425, 16240402, 71067340, 11859571, 67032666, 57451910, 6424332, 39434797, 32161947, 10773568, 468353, 81327700, 27524361, 11924421, 10157539, 51628076, 2208874, 16445064, 10462239, 13113493, 16010970, 2071244, 36115993, 3776450, 13349259, 47563719, 52808075, 34132923, 31987939, 22494382, 4793263, 4631717, 3227971, 1532728, 1779997, 525616, 1346016, 2815660, 2355857, 1437473, 2301331, 625673, 814615, 1815309, 4743130, 3387297, 2520030, 1698033, 4551099, 7630925, 10954669, 541070, 5911542, 4984844, 3203231, 4697258, 15953416, 7742204, 2851346, 10498516, 1014879, 14343371, 13722832, 4977544, 8335967, 4797116, 9393526, 5009392, 3294064, 1775251, 4200339, 8723842, 11076453, 4035617, 2711507, 7878530, 16574075, 453036, 13100874, 2262100, 263934, 9486970, 1802608, 4408482, 11722678, 1641437, 1699704, 1890793, 21749809, 1833567, 1480095, 949497, 1667322, 17404379, 1003395, 888840, 1208110, 1430164, 3172508, 1878465, 1237329, 1363972, 1309953, 871043, 1283374, 1477162], "NumGC":17770} Allocated: 11198725336, active: 11200122880, metadata: 45751184 (n_thp 0), resident: 11710238720, mapped: 11728867328, retained: 5828489216 2021-09-13 16:41:40 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: starting... frag 63, data: 49356544, used: 136122500 log:(4544380778 - 4680503296) head:4530353220 2021-09-13 16:41:40 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: completed... frag 30, data: 47072396, used: 67755160, disk: 97075991, relocated: 187309, retries: 31198, skipped: 32874, cleaned:3570140509, log:(8114521287 - 8182669312), disk:(8085593321-8182669312), activeFragRatio:30, activeMaxFragRatio:80, run:163 duration:137745 ms elapsed: 301740 ms 2021-09-13 16:41:41 [plasma] [i] Plasma: mvccPurger: finished... purge_ratio 6.15459 2021-09-13 16:41:48 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: starting... frag 80, data: 119047096, used: 625656798 log:(37190870591 - 37816848384) head:37153197413 2021-09-13 16:41:48 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: completed... frag 95, data: 59918133, used: 1342145046, disk: 1378357182, relocated: 262325, retries: 15499, skipped: 11242, cleaned:3671145348, log:(40862015939 - 42205196288), disk:(40826839106-42205196288), activeFragRatio:30, activeMaxFragRatio:80, run:6 duration:295664 ms elapsed: 308297 ms 2021-09-13 16:41:27 [ss-workload] 2 14% 14% 70% 2021-09-13 16:41:27 [ss-workload] 3 14% 14% 70% 2021-09-13 16:41:27 [ss-workload] 4 3% 3% 92% 2021-09-13 16:41:27 [ss-workload] 5 3% 3% 92% 2021-09-13 16:41:27 [ss-workload] 6 3% 3% 92% 2021-09-13 16:41:27 [ss-workload] 7 18% 18% 62% 2021-09-13 16:41:27 [ss-workload] 8 18% 18% 62% 2021-09-13 16:41:27 [ss-workload] 9 18% 18% 62% 2021-09-13 16:41:27 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 16:41:27 [ss-diag] 0 128471 126883 0 0 177 22 11601996 208 117695876 13577932 1.000000 29.369351 1.000000 0:0:0 2021-09-13 16:41:27 [ss-diag] 1 297589 295951 0 0 263 34 17239124 208 120973100 13577788 1.000000 4.374395 1.000000 0:0:0 2021-09-13 16:41:27 [ss-diag] 2 201008 199702 0 0 2780 17 182223440 66 89884420 4263412 1.000000 0.993978 1.000000 0:0:0 2021-09-13 16:41:27 [ss-diag] 3 171790 170125 0 0 263 33 17239124 208 122734792 13577580 1.000000 0.993978 1.000000 0:0:0 2021-09-13 16:41:27 [ss-diag] 4 5766572 5764988 0 0 4791 619 314040468 208 117422068 13577836 1.000000 1.955193 1.000000 0:0:0 2021-09-13 16:41:27 [ss-diag] 5 5744196 5742586 0 0 5225 668 342488300 208 119125308 13578028 1.000000 0.993978 1.000000 0:0:0 2021-09-13 16:41:27 [ss-diag] 6 5743297 5741683 0 0 4798 613 314499304 208 119387468 13577692 1.000000 0.993978 1.000000 0:0:0 2021-09-13 16:41:27 [ss-diag] 7 544470 542796 0 0 246 32 16124808 208 123330556 13577644 1.000000 3.990027 1.000000 0:0:0 2021-09-13 16:41:27 [ss-diag] 8 342959 341287 0 0 421 55 27595708 208 123191060 13577836 1.000000 0.993978 1.000000 0:0:0 2021-09-13 16:41:27 [ss-diag] 9 517034 515381 0 0 458 58 30020984 208 121950936 13577804 1.000000 0.993978 1.000000 0:0:0 2021-09-13 16:42:04 [gsi-test] [] { "memory_quota": 1516282026, "count": 31270, "compacts": 2279, "purges": 126, "splits": 207, "merges": 0, "inserts": 31270, "deletes": 0, "compact_conflicts": 259, "split_conflicts": 45, "merge_conflicts": 0, "insert_conflicts": 3300, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 7, "memory_size": 121235580, "memory_size_index": 13577932, "allocated": 8752547840, "freed": 8631312260, "reclaimed": 8628886432, "reclaim_pending": 2425828, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577932, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1642, "num_rec_allocs": 130910, "num_rec_frees": 129268, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 2049685960, "lss_data_size": 42353847, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 69432536, "est_recovery_size": 36662544, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 94580, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 63310, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.89423, "currSn": 2330, "gcSn": 2328, "gcSnIntervals": "[0 1144 1716 2261 2329]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 28, "num_trim_wctxs": 120, "num_readers": 9, "num_writers": 10, "page_bytes": 13765080, "page_cnt": 26, "page_itemcnt": 210, "avg_item_size": 65548, "avg_page_size": 529426, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 123539512, "page_bytes_marshalled":7862933430, "page_bytes_compressed":4280040977, "compression_ratio": 1.83712, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 24192, "num_total_smr": 4, "num_active_smr": 1, "num_pending_smr": 2, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 581693321, "lss_data_size": 58425791, "lss_used_space": 96278359, "lss_disk_size": 210534400, "lss_fragmentation": 39, "lss_num_reads": 7948, "lss_read_bs": 8290559642, "lss_blk_read_bs": 8352784384, "bytes_written": 8330706944, "bytes_incoming": 2049685960, "write_amp": 29.37, "write_amp_avg": 3.67, "lss_gc_num_reads": 7948, "lss_gc_reads_bs": 8290559642, "lss_blk_gc_reads_bs": 8352784384, "lss_gc_status": "frag 30, data: 42353847, used: 60925923, disk: 83640388, relocated: 13479, retries: 3152, skipped: 3748, cleaned:77612654, log:(7456103423 - 7517351936), disk:(7433711548-7517351936), activeFragRatio:30, activeMaxFragRatio:80, run:16 duration:10597 ms elapsed: 28051 ms", "lss_head_offset": 7456103423, "lss_tail_offset": 7517351936, "num_sctxs": 39, "num_free_sctxs": 31, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:42:05 [gsi-test] [] { "memory_quota": 1516282026, "count": 30728, "compacts": 6917, "purges": 481, "splits": 207, "merges": 0, "inserts": 30728, "deletes": 0, "compact_conflicts": 188, "split_conflicts": 106, "merge_conflicts": 0, "insert_conflicts": 2992, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 122415844, "memory_size_index": 13577788, "allocated": 20122414940, "freed": 19999999096, "reclaimed": 19999999096, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577788, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1660, "num_rec_allocs": 299834, "num_rec_frees": 298174, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 2014158944, "lss_data_size": 45500419, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 75834031, "est_recovery_size": 39613262, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 94803, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 64075, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.98077, "currSn": 2327, "gcSn": 2324, "gcSnIntervals": "[0 1155 1728 2277 2325]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 22, "num_trim_wctxs": 134, "num_readers": 5, "num_writers": 8, "page_bytes": 18550084, "page_cnt": 37, "page_itemcnt": 283, "avg_item_size": 65548, "avg_page_size": 501353, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 141109240, "page_bytes_marshalled":7662427980, "page_bytes_compressed":4171259719, "compression_ratio": 1.83696, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 24448, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 404911132, "lss_data_size": 62162475, "lss_used_space": 104992742, "lss_disk_size": 229081088, "lss_fragmentation": 40, "lss_num_reads": 31471, "lss_read_bs": 11996014535, "lss_blk_read_bs": 12182294528, "bytes_written": 11906023424, "bytes_incoming": 2014158944, "write_amp": 4.37, "write_amp_avg": 4.10, "lss_gc_num_reads": 31471, "lss_gc_reads_bs": 11996014535, "lss_blk_gc_reads_bs": 12182294528, "lss_gc_status": "frag 43, data: 41786328, used: 74501651 log:(8186954552 - 8262205440) head:8174418874", "lss_head_offset": 8195133256, "lss_tail_offset": 8262205440, "num_sctxs": 31, "num_free_sctxs": 23, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:42:05 [gsi-test] [] { "memory_quota": 1516282026, "count": 30300, "compacts": 745, "purges": 333, "splits": 125, "merges": 0, "inserts": 30300, "deletes": 0, "compact_conflicts": 249, "split_conflicts": 25, "merge_conflicts": 0, "insert_conflicts": 3029, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 40, "memory_size": 98673148, "memory_size_index": 8198820, "allocated": 13423372416, "freed": 13324699268, "reclaimed": 13322928984, "reclaim_pending": 1770284, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 8198820, "freed_index": 0, "reclaimed_index": 0, "num_pages": 126, "items_count": 208, "total_records": 1380, "num_rec_allocs": 203898, "num_rec_frees": 202518, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1986104400, "lss_data_size": 41698297, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 595689957, "est_recovery_size": 32407854, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 90878, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 60578, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 6.63462, "currSn": 2181, "gcSn": 2176, "gcSnIntervals": "[0 1047 1626 2144 2177]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 25, "num_trim_wctxs": 126, "num_readers": 4, "num_writers": 13, "page_bytes": 41950720, "page_cnt": 28, "page_itemcnt": 640, "avg_item_size": 65548, "avg_page_size": 1498240, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 134144800, "page_bytes_marshalled":4988240318, "page_bytes_compressed":2715135167, "compression_ratio": 1.83720, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12368, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 773994742, "lss_data_size": 128455271, "lss_used_space": 1881487498, "lss_disk_size": 1919049728, "lss_fragmentation": 93, "lss_num_reads": 107278, "lss_read_bs": 47865659439, "lss_blk_read_bs": 49411891200, "bytes_written": 49499234304, "bytes_incoming": 41903197700, "write_amp": 0.99, "write_amp_avg": 1.02, "lss_gc_num_reads": 107278, "lss_gc_reads_bs": 47865659439, "lss_blk_gc_reads_bs": 49411891200, "lss_gc_status": "frag 95, data: 59918133, used: 1342145046 log:(40862015939 - 42205196288) head:40826839106", "lss_head_offset": 40971115688, "lss_tail_offset": 42541432832, "num_sctxs": 73, "num_free_sctxs": 58, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:42:05 [gsi-test] [] { "memory_quota": 1516282026, "count": 29996, "compacts": 5646, "purges": 169, "splits": 207, "merges": 0, "inserts": 29996, "deletes": 0, "compact_conflicts": 249, "split_conflicts": 20, "merge_conflicts": 0, "insert_conflicts": 3233, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 9, "memory_size": 124897180, "memory_size_index": 13577580, "allocated": 11789011252, "freed": 11664114072, "reclaimed": 11662212692, "reclaim_pending": 1901380, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 1, "allocated_index": 13577580, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1698, "num_rec_allocs": 173880, "num_rec_frees": 172182, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1966177808, "lss_data_size": 14826313, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 185328912, "est_recovery_size": 39686698, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 91797, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 61801, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 8.16346, "currSn": 2184, "gcSn": 2180, "gcSnIntervals": "[0 1050 1615 2120 2181]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 25, "num_trim_wctxs": 122, "num_readers": 7, "num_writers": 8, "page_bytes": 17239124, "page_cnt": 33, "page_itemcnt": 263, "avg_item_size": 65548, "avg_page_size": 522397, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 135200288, "page_bytes_marshalled":5157819082, "page_bytes_compressed":2807609597, "compression_ratio": 1.83709, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 14656, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 773994742, "lss_data_size": 133710230, "lss_used_space": 1871975964, "lss_disk_size": 1920892928, "lss_fragmentation": 92, "lss_num_reads": 107338, "lss_read_bs": 47877056998, "lss_blk_read_bs": 49423507456, "bytes_written": 49501077504, "bytes_incoming": 41912439968, "write_amp": 0.99, "write_amp_avg": 1.02, "lss_gc_num_reads": 107338, "lss_gc_reads_bs": 47877056998, "lss_blk_gc_reads_bs": 49423507456, "lss_gc_status": "frag 95, data: 59918133, used: 1342145046 log:(40862015939 - 42205196288) head:40826839106", "lss_head_offset": 40978044838, "lss_tail_offset": 42541432832, "num_sctxs": 73, "num_free_sctxs": 60, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:42:05 [gsi-test] [] { "memory_quota": 1516282026, "count": 241950, "compacts": 836184, "purges": 0, "splits": 207, "merges": 0, "inserts": 241950, "deletes": 0, "compact_conflicts": 246, "split_conflicts": 28, "merge_conflicts": 0, "insert_conflicts": 2421, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 25, "memory_size": 118470708, "memory_size_index": 13577836, "allocated": 435679104668, "freed": 435560633960, "reclaimed": 435557224760, "reclaim_pending": 3409200, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577836, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1600, "num_rec_allocs": 5813007, "num_rec_frees": 5811407, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 15859338600, "lss_data_size": 5862606, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 146565150, "est_recovery_size": 74038014, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 4111761, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 3869811, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.69231, "currSn": 42910, "gcSn": 42893, "gcSnIntervals": "[0 20176 30592 41471 42894]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 33, "num_trim_wctxs": 162, "num_readers": 10, "num_writers": 14, "page_bytes": 366872156, "page_cnt": 715, "page_itemcnt": 5597, "avg_item_size": 65548, "avg_page_size": 513107, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 130868792, "page_bytes_marshalled":44232311588, "page_bytes_compressed":24099885806, "compression_ratio": 1.83537, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12672, "num_total_smr": 4, "num_active_smr": 1, "num_pending_smr": 2, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 117708848, "lss_data_size": 7968974, "lss_used_space": 223923562, "lss_disk_size": 318160896, "lss_fragmentation": 96, "lss_num_reads": 541457, "lss_read_bs": 85155344293, "lss_blk_read_bs": 86223667200, "bytes_written": 85613527040, "bytes_incoming": 15859338600, "write_amp": 1.96, "write_amp_avg": 1.71, "lss_gc_num_reads": 541457, "lss_gc_reads_bs": 85155344293, "lss_blk_gc_reads_bs": 86223667200, "lss_gc_status": "frag 94, data: 7470130, used: 137130425 log:(26965962570 - 27103735808) head:26934961607", "lss_head_offset": 27019707182, "lss_tail_offset": 27165802496, "num_sctxs": 43, "num_free_sctxs": 35, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:42:05 [gsi-test] [] { "memory_quota": 1516282026, "count": 239116, "compacts": 828009, "purges": 0, "splits": 207, "merges": 0, "inserts": 239116, "deletes": 0, "compact_conflicts": 280, "split_conflicts": 21, "merge_conflicts": 0, "insert_conflicts": 2197, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 40, "memory_size": 118863084, "memory_size_index": 13578028, "allocated": 433019431388, "freed": 432900568304, "reclaimed": 432893749904, "reclaim_pending": 6818400, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13578028, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1606, "num_rec_allocs": 5780504, "num_rec_frees": 5778898, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 15673575568, "lss_data_size": 400665, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 5008312, "est_recovery_size": 68999618, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 4124750, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 3885634, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.72115, "currSn": 41555, "gcSn": 41537, "gcSnIntervals": "[0 19143 30229 40009 41538]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 31, "num_trim_wctxs": 169, "num_readers": 12, "num_writers": 14, "page_bytes": 410068288, "page_cnt": 794, "page_itemcnt": 6256, "avg_item_size": 65548, "avg_page_size": 516458, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 132703272, "page_bytes_marshalled":9588357704, "page_bytes_compressed":5223964758, "compression_ratio": 1.83546, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 11712, "num_total_smr": 2, "num_active_smr": 5, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 773994742, "lss_data_size": 133710230, "lss_used_space": 1871975964, "lss_disk_size": 1920892928, "lss_fragmentation": 92, "lss_num_reads": 107338, "lss_read_bs": 47877056998, "lss_blk_read_bs": 49423507456, "bytes_written": 49501077504, "bytes_incoming": 41912439968, "write_amp": 0.99, "write_amp_avg": 1.02, "lss_gc_num_reads": 107338, "lss_gc_reads_bs": 47877056998, "lss_blk_gc_reads_bs": 49423507456, "lss_gc_status": "frag 95, data: 59918133, used: 1342145046 log:(40862015939 - 42205196288) head:40826839106", "lss_head_offset": 40978044838, "lss_tail_offset": 42541432832, "num_sctxs": 73, "num_free_sctxs": 59, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:42:05 [gsi-test] [] { "memory_quota": 1516282026, "count": 238988, "compacts": 827132, "purges": 0, "splits": 207, "merges": 0, "inserts": 238988, "deletes": 0, "compact_conflicts": 298, "split_conflicts": 39, "merge_conflicts": 0, "insert_conflicts": 2406, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 51, "memory_size": 122928004, "memory_size_index": 13577692, "allocated": 432934961640, "freed": 432812033636, "reclaimed": 432810853532, "reclaim_pending": 1180104, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577692, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1668, "num_rec_allocs": 5780063, "num_rec_frees": 5778395, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 15665185424, "lss_data_size": 2936185, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 36702312, "est_recovery_size": 58704000, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 4123251, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 3884263, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 8.01923, "currSn": 41341, "gcSn": 41316, "gcSnIntervals": "[0 18901 29933 39667 41317]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 31, "num_trim_wctxs": 178, "num_readers": 9, "num_writers": 13, "page_bytes": 375196752, "page_cnt": 733, "page_itemcnt": 5724, "avg_item_size": 65548, "avg_page_size": 511864, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 129949584, "page_bytes_marshalled":9061604454, "page_bytes_compressed":4936980537, "compression_ratio": 1.83545, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12160, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 773994742, "lss_data_size": 133710230, "lss_used_space": 1871975964, "lss_disk_size": 1920892928, "lss_fragmentation": 92, "lss_num_reads": 107338, "lss_read_bs": 47877056998, "lss_blk_read_bs": 49423507456, "bytes_written": 49501077504, "bytes_incoming": 41912439968, "write_amp": 0.99, "write_amp_avg": 1.02, "lss_gc_num_reads": 107338, "lss_gc_reads_bs": 47877056998, "lss_blk_gc_reads_bs": 49423507456, "lss_gc_status": "frag 95, data: 59918133, used: 1342145046 log:(40862015939 - 42205196288) head:40826839106", "lss_head_offset": 40978044838, "lss_tail_offset": 42541432832, "num_sctxs": 73, "num_free_sctxs": 59, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:42:05 [gsi-test] [] { "memory_quota": 1516282026, "count": 52596, "compacts": 70497, "purges": 0, "splits": 207, "merges": 0, "inserts": 52596, "deletes": 0, "compact_conflicts": 305, "split_conflicts": 30, "merge_conflicts": 0, "insert_conflicts": 4181, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 5, "memory_size": 124510500, "memory_size_index": 13577644, "allocated": 40626794948, "freed": 40502284448, "reclaimed": 40497367108, "reclaim_pending": 4917340, "reclaim_list_size": 655608, "reclaim_list_count": 1, "reclaim_threshold": 0, "allocated_index": 13577644, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 207, "total_records": 1692, "num_rec_allocs": 549138, "num_rec_frees": 547446, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 3447562608, "lss_data_size": 45280717, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 75467861, "est_recovery_size": 37514990, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 144736, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 92140, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 8.17391, "currSn": 2765, "gcSn": 2761, "gcSnIntervals": "[0 1308 1978 2664 2762]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 19, "num_trim_wctxs": 119, "num_readers": 3, "num_writers": 9, "page_bytes": 34740440, "page_cnt": 67, "page_itemcnt": 530, "avg_item_size": 65548, "avg_page_size": 518514, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 131927032, "page_bytes_marshalled":15727627520, "page_bytes_compressed":8568433282, "compression_ratio": 1.83553, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 21312, "num_total_smr": 2, "num_active_smr": 3, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 64454699, "lss_data_size": 61614925, "lss_used_space": 102729423, "lss_disk_size": 236728320, "lss_fragmentation": 40, "lss_num_reads": 427418, "lss_read_bs": 59653974950, "lss_blk_read_bs": 61365305344, "bytes_written": 59963617280, "bytes_incoming": 3447562608, "write_amp": 3.99, "write_amp_avg": 2.53, "lss_gc_num_reads": 427418, "lss_gc_reads_bs": 59653974950, "lss_blk_gc_reads_bs": 61365305344, "lss_gc_status": "frag 30, data: 45995744, used: 65996565, disk: 95392046, relocated: 714157, retries: 1223, skipped: 12819, cleaned:303772702, log:(8661922472 - 8728883200), disk:(8633491154-8728883200), activeFragRatio:30, activeMaxFragRatio:80, run:26 duration:34705 ms elapsed: 60585 ms", "lss_head_offset": 8661922472, "lss_tail_offset": 8728883200, "num_sctxs": 33, "num_free_sctxs": 23, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:42:07 [gsi-test] [] { "memory_quota": 1516282026, "count": 51944, "compacts": 17268, "purges": 413, "splits": 207, "merges": 0, "inserts": 51944, "deletes": 0, "compact_conflicts": 492, "split_conflicts": 49, "merge_conflicts": 0, "insert_conflicts": 6157, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 9, "memory_size": 121879364, "memory_size_index": 13577836, "allocated": 23815341952, "freed": 23693462588, "reclaimed": 23687037492, "reclaim_pending": 6425096, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577836, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1652, "num_rec_allocs": 345805, "num_rec_frees": 344153, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 3404825312, "lss_data_size": 3757560, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 53679428, "est_recovery_size": 50113470, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 137897, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 85953, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 7.94231, "currSn": 2504, "gcSn": 2500, "gcSnIntervals": "[0 1170 1832 2414 2501]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 25, "num_trim_wctxs": 114, "num_readers": 4, "num_writers": 12, "page_bytes": 35199276, "page_cnt": 69, "page_itemcnt": 537, "avg_item_size": 65548, "avg_page_size": 510134, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 127199960, "page_bytes_marshalled":5590968308, "page_bytes_compressed":3043707277, "compression_ratio": 1.83689, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12416, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 773994742, "lss_data_size": 126918181, "lss_used_space": 1877208179, "lss_disk_size": 1950224384, "lss_fragmentation": 93, "lss_num_reads": 107436, "lss_read_bs": 47901395334, "lss_blk_read_bs": 49449709568, "bytes_written": 49530408960, "bytes_incoming": 41944558488, "write_amp": 0.99, "write_amp_avg": 1.01, "lss_gc_num_reads": 107436, "lss_gc_reads_bs": 47901395334, "lss_blk_gc_reads_bs": 49449709568, "lss_gc_status": "frag 95, data: 59918133, used: 1342145046 log:(40862015939 - 42205196288) head:40826839106", "lss_head_offset": 40995971456, "lss_tail_offset": 42562072576, "num_sctxs": 74, "num_free_sctxs": 66, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13T16:42:08.817-07:00 [Info] memstats {"Alloc":495148328, "TotalAlloc":3382044568960, "Sys":976283896, "Lookups":0, "Mallocs":212964789,"Frees":212933756, "HeapAlloc":495148328, "HeapSys":934084608, "HeapIdle":436101120, "HeapInuse":497983488,"HeapReleased":359325696, "HeapObjects":31033,"GCSys":30998528, "LastGC":1631576528816889140,"PauseTotalNs":145450326447, "PauseNs":[961048, 1623361, 1840753, 1818428, 6951101, 1174436, 8720444, 1270457, 2376063, 946336, 1437605, 4366432, 6655743, 1516392, 5310728, 1901866, 2473085, 1450012, 793784, 1560047, 5981778, 810216, 2407916, 1503473, 981438, 3877336, 469779, 1303354, 6617693, 9538523, 2987833, 10936590, 8539234, 7448836, 9421995, 8255246, 11478752, 13353823, 805717, 1794590, 5244740, 4349363, 12837732, 2474014, 10974345, 9408958, 1135132, 2789764, 13296358, 12057641, 6507014, 26756963, 14495792, 2152306, 1021163, 2774170, 7201567, 5809794, 9156000, 9735234, 4313982, 5084100, 4197423, 514226, 729722, 1922272, 1128672, 1668892, 1793401, 4097343, 1517096, 7493429, 7599953, 6307936, 3881504, 23297240, 9705901, 5515227, 12157340, 4871875, 3294641, 4742115, 9385197, 10466630, 6871987, 9012534, 21810902, 2841462, 15413736, 2279081, 6687343, 833230, 1647530, 1789629, 1597669, 4844333, 772672, 4283314, 1093011, 1141662, 1626979, 1100191, 1560076, 947425, 1375894, 1066440, 3047861, 2275495, 1825102, 1335992, 9289125, 965061, 6897257, 3023666, 895204, 1601703, 4196720, 2079953, 1944211, 1440804, 1227628, 1240794, 4667952, 1714404, 1724924, 999542, 1113457, 669878, 3794889, 1587547, 1656911, 1787903, 1989609, 4141209, 1490444, 4149502, 1366151, 1222386, 837692, 4655301, 1985561, 7711832, 10360646, 1823681, 1364015, 4636263, 5455714, 998275, 6226352, 1018823, 1507627, 724483, 5241651, 6592549, 1328451, 1243038, 570380, 1393544, 3073771, 702901, 788513], "NumGC":17931} Allocated: 3949801400, active: 3951042560, metadata: 47935152 (n_thp 0), resident: 4237971456, mapped: 4254502912, retained: 15181901824 2021-09-13 16:42:09 [gsi-test] [] { "memory_quota": 1516282026, "count": 49196, "compacts": 67042, "purges": 0, "splits": 207, "merges": 0, "inserts": 49196, "deletes": 0, "compact_conflicts": 311, "split_conflicts": 32, "merge_conflicts": 0, "insert_conflicts": 3957, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 5, "memory_size": 123456868, "memory_size_index": 13577804, "allocated": 38472330524, "freed": 38348873656, "reclaimed": 38348218048, "reclaim_pending": 655608, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577804, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1676, "num_rec_allocs": 519776, "num_rec_frees": 518100, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 3224699408, "lss_data_size": 27360784, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 390868342, "est_recovery_size": 35227376, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 136603, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 87407, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 8.05769, "currSn": 2586, "gcSn": 2579, "gcSnIntervals": "[0 1176 1854 2478 2580]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 23, "num_trim_wctxs": 137, "num_readers": 3, "num_writers": 10, "page_bytes": 27726804, "page_cnt": 53, "page_itemcnt": 423, "avg_item_size": 65548, "avg_page_size": 523147, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 132048384, "page_bytes_marshalled":6379326738, "page_bytes_compressed":3475593137, "compression_ratio": 1.83546, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 16576, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 777326872, "lss_data_size": 113140542, "lss_used_space": 1879437194, "lss_disk_size": 1981063168, "lss_fragmentation": 93, "lss_num_reads": 107535, "lss_read_bs": 47930244389, "lss_blk_read_bs": 49479892992, "bytes_written": 49561247744, "bytes_incoming": 41983821740, "write_amp": 0.99, "write_amp_avg": 1.01, "lss_gc_num_reads": 107535, "lss_gc_reads_bs": 47930244389, "lss_blk_gc_reads_bs": 49479892992, "lss_gc_status": "frag 95, data: 59918133, used: 1342145046 log:(40862015939 - 42205196288) head:40826839106", "lss_head_offset": 41021403172, "lss_tail_offset": 42582642688, "num_sctxs": 74, "num_free_sctxs": 58, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } Mon Sep 13 16:42:15 PDT 2021 RSS = 4211612 2021-09-13 16:42:21 [gsi-test] [] Stop and verify... 2021-09-13 16:42:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper stopped 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:42:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : All daemons stopped 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper stopped 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:42:22 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:42:22 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : All daemons stopped 2021-09-13 16:42:23 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper stopped 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 16:42:23 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : All daemons stopped 2021-09-13 16:42:23 [gsi-test] [0] Stop and verify... 2021-09-13 16:42:23 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 16:42:24 [gsi-test] [0] Stop and verify... OK 2021-09-13 16:42:24 [gsi-test] [1] Stop and verify... 2021-09-13 16:42:26 [gsi-test] [1] Stop and verify... OK 2021-09-13 16:42:26 [gsi-test] [2] Stop and verify... 2021-09-13T16:42:38.937-07:00 [Info] memstats {"Alloc":388483584, "TotalAlloc":3402582594944, "Sys":976283896, "Lookups":0, "Mallocs":214944775,"Frees":214917348, "HeapAlloc":388483584, "HeapSys":934871040, "HeapIdle":543383552, "HeapInuse":391487488,"HeapReleased":432660480, "HeapObjects":27427,"GCSys":30998528, "LastGC":1631576558859098367,"PauseTotalNs":145653127526, "PauseNs":[1777214, 3441544, 4436610, 1112009, 886144, 2176280, 2052775, 6348034, 4907546, 1395601, 851687, 1045249, 8050666, 1252058, 2205582, 5968348, 886912, 1092965, 5035510, 1398826, 3768573, 2520451, 1595542, 1016143, 1484818, 1489260, 1555524, 2442702, 2453034, 731206, 3881996, 1354982, 1383908, 1847566, 2165740, 6417662, 1536308, 1458991, 9705722, 1612866, 2352420, 1382837, 1093882, 4508985, 5700841, 4539775, 6176044, 4436280, 7779214, 1727795, 3270107, 2607751, 5654724, 1980716, 1879967, 2374767, 1356921, 2910255, 1972301, 3392593, 5849636, 2916134, 6031660, 461881, 906899, 457201, 563525, 353306, 675959, 1678996, 363692, 651952, 903447, 533565, 554424, 1526634, 514895, 1067256, 434582, 425357, 375278, 415872, 571556, 724643], "NumGC":18015} Allocated: 3748614456, active: 3749830656, metadata: 47935152 (n_thp 0), resident: 3848282112, mapped: 3864813568, retained: 15571591168 2021-09-13 16:43:04 [ss-workload] instance inse% dele% read% 2021-09-13 16:43:04 [ss-workload] 0 14% 14% 70% 2021-09-13 16:43:04 [ss-workload] 1 14% 14% 70% 2021-09-13T16:43:08.966-07:00 [Info] memstats {"Alloc":436636984, "TotalAlloc":3410314606488, "Sys":976283896, "Lookups":0, "Mallocs":215578741,"Frees":215547090, "HeapAlloc":436636984, "HeapSys":934969344, "HeapIdle":495403008, "HeapInuse":439566336,"HeapReleased":434577408, "HeapObjects":31651,"GCSys":30900224, "LastGC":1631576588125181578,"PauseTotalNs":145687598344, "PauseNs":[361342, 995654, 1016787, 376246, 13786140, 345600, 410090, 433890, 373819, 349521, 316727, 493119, 407131, 422376, 410757, 1397658, 736395, 773669, 632093, 490456, 881221, 1272618, 370487, 486712, 484697, 441510, 950999, 641725, 684017, 1617068, 462033, 431297, 444443, 772521], "NumGC":18049} Allocated: 3754863776, active: 3756089344, metadata: 47935152 (n_thp 0), resident: 3858849792, mapped: 3875381248, retained: 15561023488 Mon Sep 13 16:43:15 PDT 2021 RSS = 3944328 2021-09-13T16:43:39.001-07:00 [Info] memstats {"Alloc":424772504, "TotalAlloc":3418164916096, "Sys":976283896, "Lookups":0, "Mallocs":216220026,"Frees":216189891, "HeapAlloc":424772504, "HeapSys":934969344, "HeapIdle":507191296, "HeapInuse":427778048,"HeapReleased":434577408, "HeapObjects":30135,"GCSys":30900224, "LastGC":1631576618298545628,"PauseTotalNs":145714081206, "PauseNs":[552967, 738429, 474159, 426661, 1003348, 591460, 1039201, 510175, 925530, 414948, 781462, 507405, 564744, 757633, 1827082, 401891, 815464, 1054135, 579001, 329377, 784361, 1162994, 410391, 464091, 780460, 612190, 384469, 716298, 1083776, 621659, 1733700, 1013388, 1041367, 892667, 485979], "NumGC":18084} Allocated: 3746063368, active: 3747274752, metadata: 47935152 (n_thp 0), resident: 3849674752, mapped: 3866206208, retained: 15570198528 2021-09-13T16:44:09.043-07:00 [Info] memstats {"Alloc":415259928, "TotalAlloc":3425824295032, "Sys":976283896, "Lookups":0, "Mallocs":216845327,"Frees":216815600, "HeapAlloc":415259928, "HeapSys":934969344, "HeapIdle":516628480, "HeapInuse":418340864,"HeapReleased":434577408, "HeapObjects":29727,"GCSys":30900224, "LastGC":1631576648340331412,"PauseTotalNs":145741601803, "PauseNs":[404990, 532395, 456136, 451317, 356165, 2124976, 1120284, 396196, 471237, 523565, 2808003, 959473, 805850, 631826, 620728, 593911, 1029920, 1334828, 454796, 472378, 957656, 481369, 577610, 1061258, 791886, 447384, 431980, 1257316, 489557, 1434158, 607696, 804692, 398930, 1230131], "NumGC":18118} Allocated: 3750935168, active: 3752157184, metadata: 47935152 (n_thp 0), resident: 3816906752, mapped: 3833438208, retained: 15602966528 Mon Sep 13 16:44:15 PDT 2021 RSS = 3944080 2021-09-13T16:44:39.081-07:00 [Info] memstats {"Alloc":381784304, "TotalAlloc":3433572606432, "Sys":976283896, "Lookups":0, "Mallocs":217476791,"Frees":217449960, "HeapAlloc":381784304, "HeapSys":934969344, "HeapIdle":549945344, "HeapInuse":385024000,"HeapReleased":433242112, "HeapObjects":26831,"GCSys":30900224, "LastGC":1631576678461716524,"PauseTotalNs":145772741844, "PauseNs":[1106864, 613466, 470981, 408101, 1306627, 315133, 572921, 1891858, 427027, 852913, 685578, 482233, 1380794, 431405, 426219, 1851040, 1210609, 1363265, 443750, 1123544, 844574, 372446, 1074178, 1338593, 1271444, 523040, 881067, 1235926, 624875, 442926, 1672819, 1402200, 1009561, 1082064], "NumGC":18152} Allocated: 3757990944, active: 3759202304, metadata: 47935152 (n_thp 0), resident: 3846037504, mapped: 3863355392, retained: 15573049344 2021-09-13T16:45:09.116-07:00 [Info] memstats {"Alloc":358299856, "TotalAlloc":3441436734488, "Sys":976283896, "Lookups":0, "Mallocs":218119755,"Frees":218095047, "HeapAlloc":358299856, "HeapSys":934969344, "HeapIdle":573325312, "HeapInuse":361644032,"HeapReleased":433635328, "HeapObjects":24708,"GCSys":30900224, "LastGC":1631576708604047799,"PauseTotalNs":145798465833, "PauseNs":[419231, 481349, 423927, 533503, 1249223, 1932111, 1085157, 1123157, 415855, 837704, 704962, 878814, 439667, 561923, 749120, 981687, 958763, 1442921, 543774, 386214, 507622, 469056, 468645, 1040763, 448719, 1249635, 483606, 551611, 573484, 957194, 414145, 503377, 315101, 404341, 1187628], "NumGC":18187} Allocated: 3751436528, active: 3752648704, metadata: 47935152 (n_thp 0), resident: 3817136128, mapped: 3833667584, retained: 15602737152 Mon Sep 13 16:45:15 PDT 2021 RSS = 3949124 2021-09-13 16:45:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 158, "num_active_sctxs": 88, "num_free_sctxs": 114, "num_trim_sctxs": 9, "num_wctxs": 288, "num_free_wctxs": 230, "num_active_wctxs": 58, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 112, "cpu": 8.63400, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3862814720, "memory_jemalloc_allocated": 3758809808, "memory_in_use": 3862814720, "memory_stats_size_total": 2119465009, "memory_stats_size_global": 1355421736, "memory_stats_size_page": 1221954952, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 138544, "reclaim_pending": 4523812, "buf_memused": 759176421, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 1, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16598, "total_ops": 1016786, "compacts": 2768110, "purges": 1522, "lss_data_size": 238935280, "lss_used_space": 1135067446, "lss_fragmentation": 78, "lss_num_reads": 733988, "lss_read_bs": 139464794568, "lss_blk_read_bs": 142461091840, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 140473835520, "bytes_incoming": 49798519848, "write_amp_avg": 1.52, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16598, "resident_ratio": 100.00000, "cleaner_num_reads": 733988, "cleaner_read_bs": 139464794568, "cleaner_blk_read_bs": 142461091840, "recovery_num_lssctx": 5, "recovery_num_sctxs": 29, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 9, "recovery_num_trim_sctxs": 9, "recovery_buf_memused": 5242880, "recovery_data_size": 29259556, "recovery_used_space": 257206580, "recovery_lss_blk_read_bs": 81016782848, "recovery_lss_read_bs": 78706116512, "recovery_lss__num_reads": 611359, "recovery_bytes_written": 79136378880, "recovery_lss_frag": 88, "recovery_cleaner_num_reads": 611359, "recovery_cleaner_read_bs": 78706116512, "recovery_cleaner_blk_read_bs": 81016782848, "mvcc_purge_ratio": 7.97981 }2021-09-13T16:45:39.168-07:00 [Info] memstats {"Alloc":323203680, "TotalAlloc":3449499252232, "Sys":976283896, "Lookups":0, "Mallocs":218779919,"Frees":218757647, "HeapAlloc":323203680, "HeapSys":934969344, "HeapIdle":608256000, "HeapInuse":326713344,"HeapReleased":434233344, "HeapObjects":22272,"GCSys":30900224, "LastGC":1631576738890435234,"PauseTotalNs":145820523740, "PauseNs":[406450, 865122, 463665, 387106, 416763, 1225096, 427857, 1026067, 736350, 499404, 440507, 385083, 941788, 570033, 475113, 494683, 487437, 426176, 385532, 1245645, 380320, 585901, 464212, 1436145, 395392, 752447, 512170, 741450, 414203, 573660, 408047, 1202008, 468134, 419683, 451073, 547185], "NumGC":18223} Allocated: 3753810920, active: 3755008000, metadata: 47935152 (n_thp 0), resident: 3850428416, mapped: 3867615232, retained: 15568789504 2021-09-13T16:46:09.202-07:00 [Info] memstats {"Alloc":315543928, "TotalAlloc":3457318083112, "Sys":976283896, "Lookups":0, "Mallocs":219418820,"Frees":219396890, "HeapAlloc":315543928, "HeapSys":934969344, "HeapIdle":615866368, "HeapInuse":319102976,"HeapReleased":433627136, "HeapObjects":21930,"GCSys":30900224, "LastGC":1631576768899929855,"PauseTotalNs":145846499663, "PauseNs":[794401, 1073149, 419208, 434274, 872008, 490228, 1126864, 718745, 1642503, 408943, 532274, 713329, 411445, 469138, 617235, 2039968, 373254, 376971, 707207, 390202, 355433, 1125624, 552814, 1416908, 1290550, 529588, 1144138, 944836, 1156440, 456303, 367622, 423483, 609130, 454412, 537296], "NumGC":18258} Allocated: 3761691528, active: 3762888704, metadata: 47935152 (n_thp 0), resident: 3865124864, mapped: 3882442752, retained: 15553961984 Mon Sep 13 16:46:15 PDT 2021 RSS = 3941208 2021-09-13T16:46:39.234-07:00 [Info] memstats {"Alloc":378816944, "TotalAlloc":3465230138368, "Sys":976283896, "Lookups":0, "Mallocs":220065461,"Frees":220038587, "HeapAlloc":378816944, "HeapSys":934969344, "HeapIdle":552869888, "HeapInuse":382099456,"HeapReleased":433979392, "HeapObjects":26874,"GCSys":30900224, "LastGC":1631576798727248160,"PauseTotalNs":145870019758, "PauseNs":[432090, 539033, 1033592, 864776, 1128774, 747647, 756919, 1358648, 632962, 1033705, 614527, 767233, 481629, 556628, 852841, 476950, 391609, 521405, 544031, 594830, 414886, 635870, 572837, 377518, 936234, 396996, 469779, 431828, 552867, 981890, 325464, 395751, 562180, 458939, 1677227], "NumGC":18293} Allocated: 3759311608, active: 3760513024, metadata: 47935152 (n_thp 0), resident: 3873431552, mapped: 3889963008, retained: 15546441728 2021-09-13T16:47:09.273-07:00 [Info] memstats {"Alloc":316051888, "TotalAlloc":3473203560456, "Sys":976283896, "Lookups":0, "Mallocs":220715981,"Frees":220691328, "HeapAlloc":316051888, "HeapSys":934871040, "HeapIdle":615563264, "HeapInuse":319307776,"HeapReleased":433045504, "HeapObjects":24653,"GCSys":30998528, "LastGC":1631576829102943296,"PauseTotalNs":145899480242, "PauseNs":[678518, 400007, 1138725, 1431860, 939332, 365619, 501840, 378061, 1194167, 861080, 520826, 425643, 1826916, 1013367, 481857, 1559931, 1125466, 536042, 721122, 794042, 621262, 395332, 1031413, 518543, 1605754, 353418, 600257, 1325191, 876111, 1239173, 764397, 649149, 1057581, 440398, 562071, 526013], "NumGC":18329} Allocated: 3754183896, active: 3755401216, metadata: 47935152 (n_thp 0), resident: 3840450560, mapped: 3856982016, retained: 15579422720 2021-09-13 16:47:12 [gsi-test] [] { "memory_quota": 1516282026, "count": 33663, "compacts": 7512, "purges": 126, "splits": 207, "merges": 0, "inserts": 33663, "deletes": 0, "compact_conflicts": 259, "split_conflicts": 45, "merge_conflicts": 0, "insert_conflicts": 3300, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 7, "memory_size": 130269764, "memory_size_index": 13577932, "allocated": 12048976604, "freed": 11918706840, "reclaimed": 11916805508, "reclaim_pending": 1901332, "reclaim_list_size": 655640, "reclaim_list_count": 1, "reclaim_threshold": 0, "allocated_index": 13577932, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 208, "total_records": 1780, "num_rec_allocs": 175992, "num_rec_frees": 174212, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 2206542324, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 819200, "est_recovery_size": 28597288, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 104586, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 70923, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 8.55769, "currSn": 2634, "gcSn": 2630, "gcSnIntervals": "[0 1144 1716 2261 2631]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 28, "num_trim_wctxs": 120, "num_readers": 9, "num_writers": 10, "page_bytes": 12847408, "page_cnt": 26, "page_itemcnt": 196, "avg_item_size": 65548, "avg_page_size": 494131, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 143061728, "page_bytes_marshalled":7970433800, "page_bytes_compressed":4338598670, "compression_ratio": 1.83710, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 11648, "num_total_smr": 3, "num_active_smr": 1, "num_pending_smr": 1, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 581693321, "lss_data_size": 16448, "lss_used_space": 102379861, "lss_disk_size": 231510016, "lss_fragmentation": 99, "lss_num_reads": 11950, "lss_read_bs": 8840204141, "lss_blk_read_bs": 8919666688, "bytes_written": 8888553472, "bytes_incoming": 2206607872, "write_amp": 3.75, "write_amp_avg": 3.44, "lss_gc_num_reads": 11950, "lss_gc_reads_bs": 8840204141, "lss_blk_gc_reads_bs": 8919666688, "lss_gc_status": "frag 30, data: 50529206, used: 72496104, disk: 76849793, relocated: 28853, retries: 6107, skipped: 7336, cleaned:134503186, log:(7512993955 - 7585882112), disk:(7509032319-7585882112), activeFragRatio:30, activeMaxFragRatio:80, run:31 duration:14052 ms elapsed: 46050 ms", "lss_head_offset": 7512993955, "lss_tail_offset": 7587061760, "num_sctxs": 39, "num_free_sctxs": 30, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 16:47:14 [gsi-test] [] { "memory_quota": 1516282026, "count": 33156, "compacts": 12263, "purges": 481, "splits": 207, "merges": 0, "inserts": 33156, "deletes": 0, "compact_conflicts": 188, "split_conflicts": 106, "merge_conflicts": 0, "insert_conflicts": 2992, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 6, "memory_size": 130138652, "memory_size_index": 13577788, "allocated": 23479489612, "freed": 23349350960, "reclaimed": 23348826464, "reclaim_pending": 524496, "reclaim_list_size": 524496, "reclaim_list_count": 1, "reclaim_threshold": 0, "allocated_index": 13577788, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 207, "total_records": 1778, "num_rec_allocs": 345724, "num_rec_frees": 343946, "num_rec_swapout": 0, "num_rec_swapin": 0, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 2173309488, "lss_data_size": 8192, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 819200, "est_recovery_size": 30236586, "lss_num_reads": 0, "lss_read_bs": 0, "lss_blk_read_bs": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "cache_hits": 104834, "cache_misses": 0, "cache_hit_ratio": 1.00000, "rlss_num_reads": 0, "rcache_hits": 71678, "rcache_misses": 0, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 8.58937, "currSn": 2635, "gcSn": 2632, "gcSnIntervals": "[0 1155 1728 2277 2633]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 19, "num_trim_wctxs": 140, "num_readers": 5, "num_writers": 6, "page_bytes": 21696388, "page_cnt": 42, "page_itemcnt": 331, "avg_item_size": 65548, "avg_page_size": 516580, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 142405512, "page_bytes_marshalled":7771370426, "page_bytes_compressed":4230601234, "compression_ratio": 1.83694, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 11648, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 404911132, "lss_data_size": 16448, "lss_used_space": 104121297, "lss_disk_size": 232275968, "lss_fragmentation": 99, "lss_num_reads": 35374, "lss_read_bs": 12534500513, "lss_blk_read_bs": 12738617344, "bytes_written": 12446089216, "bytes_incoming": 2173309488, "write_amp": 3.41, "write_amp_avg": 3.83, "lss_gc_num_reads": 35374, "lss_gc_reads_bs": 12534500513, "lss_blk_gc_reads_bs": 12738617344, "lss_gc_status": "frag 30, data: 49175074, used: 70690235, disk: 79512119, relocated: 70932, retries: 6208, skipped: 9769, cleaned:138686034, log:(8253207321 - 8324218880), disk:(8244706761-8324218880), activeFragRatio:30, activeMaxFragRatio:80, run:32 duration:11191 ms elapsed: 41065 ms", "lss_head_offset": 8253207321, "lss_tail_offset": 8328220672, "num_sctxs": 31, "num_free_sctxs": 22, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } Mon Sep 13 16:47:15 PDT 2021 RSS = 3953360 2021-09-13T16:47:39.317-07:00 [Info] memstats {"Alloc":361725720, "TotalAlloc":3481134493304, "Sys":976283896, "Lookups":0, "Mallocs":221363205,"Frees":221337894, "HeapAlloc":361725720, "HeapSys":934969344, "HeapIdle":569876480, "HeapInuse":365092864,"HeapReleased":434823168, "HeapObjects":25311,"GCSys":30900224, "LastGC":1631576858861935043,"PauseTotalNs":145928743633, "PauseNs":[531674, 533431, 842611, 426741, 1377045, 832900, 454820, 521407, 1006328, 995413, 434314, 2505960, 432430, 425770, 486065, 658203, 638170, 619817, 936596, 1430192, 533620, 1346284, 509272, 1487176, 953205, 331642, 862794, 858601, 976341, 1443177, 474101, 449028, 387312, 870653, 1690298], "NumGC":18364} Allocated: 3757544848, active: 3758809088, metadata: 47935152 (n_thp 0), resident: 3860979712, mapped: 3877511168, retained: 15558893568 2021-09-13T16:48:09.367-07:00 [Info] memstats {"Alloc":395601392, "TotalAlloc":3488780392008, "Sys":976283896, "Lookups":0, "Mallocs":221986074,"Frees":221958329, "HeapAlloc":395601392, "HeapSys":934969344, "HeapIdle":536174592, "HeapInuse":398794752,"HeapReleased":434921472, "HeapObjects":27745,"GCSys":30900224, "LastGC":1631576888649388335,"PauseTotalNs":145954003531, "PauseNs":[432433, 432300, 513806, 644186, 487103, 449154, 326971, 808206, 947025, 348393, 1058430, 1382402, 1009293, 1225251, 433365, 1202985, 454350, 351682, 336814, 648651, 419229, 547211, 448326, 924968, 999076, 450357, 413475, 1342182, 874561, 637989, 1052564, 1394090, 982618, 1280452], "NumGC":18398} Allocated: 3753580800, active: 3754876928, metadata: 47935152 (n_thp 0), resident: 3869679616, mapped: 3886211072, retained: 15550193664 Mon Sep 13 16:48:15 PDT 2021 RSS = 3946176 2021-09-13T16:48:39.407-07:00 [Info] memstats {"Alloc":389743240, "TotalAlloc":3496601200160, "Sys":976283896, "Lookups":0, "Mallocs":222625284,"Frees":222598583, "HeapAlloc":389743240, "HeapSys":934969344, "HeapIdle":541982720, "HeapInuse":392986624,"HeapReleased":434921472, "HeapObjects":26701,"GCSys":30900224, "LastGC":1631576918785915399,"PauseTotalNs":145974815123, "PauseNs":[445879, 502032, 347583, 383766, 496070, 430395, 411796, 925978, 771260, 529474, 361127, 361127, 620228, 571533, 1364870, 441200, 939844, 972805, 428244, 1458934, 505326, 408423, 719167, 597856, 452822, 376037, 384801, 474340, 520377, 1142540, 506552, 499371, 337975, 350810, 771050], "NumGC":18433} Allocated: 3757901320, active: 3759202304, metadata: 47935152 (n_thp 0), resident: 3855523840, mapped: 3872055296, retained: 15564349440 2021-09-13T16:49:09.458-07:00 [Info] memstats {"Alloc":410125720, "TotalAlloc":3504584434536, "Sys":976283896, "Lookups":0, "Mallocs":223280277,"Frees":223251431, "HeapAlloc":410125720, "HeapSys":934969344, "HeapIdle":521699328, "HeapInuse":413270016,"HeapReleased":433471488, "HeapObjects":28846,"GCSys":30900224, "LastGC":1631576948683332870,"PauseTotalNs":146002703605, "PauseNs":[803106, 1100476, 483260, 961980, 916386, 417107, 731470, 455013, 351050, 1082512, 418197, 577263, 327039, 407435, 434506, 1200016, 390079, 1888230, 831706, 1069921, 380707, 741565, 401206, 332086, 1493985, 389926, 1383193, 580008, 521307, 3154767, 417563, 935827, 558880, 478452, 1272258], "NumGC":18468} Allocated: 3758667496, active: 3759984640, metadata: 47935152 (n_thp 0), resident: 3849687040, mapped: 3866873856, retained: 15569530880 Mon Sep 13 16:49:15 PDT 2021 RSS = 3943600 2021-09-13T16:49:39.512-07:00 [Info] memstats {"Alloc":308144728, "TotalAlloc":3512549473384, "Sys":976283896, "Lookups":0, "Mallocs":223929636,"Frees":223908301, "HeapAlloc":308144728, "HeapSys":934969344, "HeapIdle":623222784, "HeapInuse":311746560,"HeapReleased":435126272, "HeapObjects":21335,"GCSys":30900224, "LastGC":1631576979205521209,"PauseTotalNs":146025639833, "PauseNs":[427989, 521223, 734850, 325280, 910722, 724569, 710500, 668415, 499874, 476520, 497095, 835945, 453779, 518363, 1329910, 422153, 418093, 472802, 420924, 423138, 463048, 1230509, 1127090, 894697, 433847, 435230, 2533809, 369399, 407053, 448285, 404474, 651180, 519546, 368432, 424304, 433181], "NumGC":18504} Allocated: 3755905352, active: 3757232128, metadata: 47935152 (n_thp 0), resident: 3845935104, mapped: 3862466560, retained: 15573938176 2021-09-13T16:50:09.549-07:00 [Info] memstats {"Alloc":407594216, "TotalAlloc":3520519351376, "Sys":976283896, "Lookups":0, "Mallocs":224582095,"Frees":224553011, "HeapAlloc":407594216, "HeapSys":934969344, "HeapIdle":524271616, "HeapInuse":410697728,"HeapReleased":435126272, "HeapObjects":29084,"GCSys":30900224, "LastGC":1631577008760562484,"PauseTotalNs":146053023185, "PauseNs":[958719, 348619, 360458, 1107272, 1388162, 377315, 368318, 1131170, 1813732, 618430, 381602, 339704, 624114, 436794, 425744, 1043563, 480346, 400441, 495866, 478667, 1064453, 1273164, 456610, 509579, 1664815, 611286, 1065044, 451479, 447272, 935852, 872112, 571057, 1158754, 790652, 1932187], "NumGC":18539} Allocated: 3754055672, active: 3755397120, metadata: 47935152 (n_thp 0), resident: 3845296128, mapped: 3861827584, retained: 15574577152 Mon Sep 13 16:50:15 PDT 2021 RSS = 3952428 2021-09-13 16:50:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 242, "num_active_sctxs": 88, "num_free_sctxs": 155, "num_trim_sctxs": 11, "num_wctxs": 290, "num_free_wctxs": 232, "num_active_wctxs": 58, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 114, "cpu": 8.51132, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3849097216, "memory_jemalloc_allocated": 3756403704, "memory_in_use": 3849097216, "memory_stats_size_total": 3306666085, "memory_stats_size_global": 1356600848, "memory_stats_size_page": 1223134064, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 137648, "reclaim_pending": 1573488, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 1, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16616, "total_ops": 1041378, "compacts": 2893882, "purges": 1522, "lss_data_size": 172364095, "lss_used_space": 2355571844, "lss_fragmentation": 92, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 218284490752, "bytes_incoming": 66648288728, "write_amp_avg": 2.30, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16616, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 45847700, "recovery_used_space": 469149315, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123284049920, "recovery_lss_frag": 90, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.98846 }2021-09-13T16:50:39.589-07:00 [Info] memstats {"Alloc":317472344, "TotalAlloc":3528443538136, "Sys":976283896, "Lookups":0, "Mallocs":225233747,"Frees":225211240, "HeapAlloc":317472344, "HeapSys":934903808, "HeapIdle":614039552, "HeapInuse":320864256,"HeapReleased":435126272, "HeapObjects":22507,"GCSys":30965760, "LastGC":1631577039415615854,"PauseTotalNs":146084174361, "PauseNs":[785970, 1080127, 964956, 432910, 1222625, 1419203, 1359656, 453670, 401901, 667635, 541563, 989601, 414587, 901667, 1672529, 712981, 599354, 479122, 669918, 384662, 1513102, 1374020, 427164, 451794, 513063, 1344079, 974688, 390643, 639343, 3480522, 815908, 358194, 890563, 748604, 473139, 601713], "NumGC":18575} Allocated: 3753511840, active: 3754872832, metadata: 47935152 (n_thp 0), resident: 3819606016, mapped: 3836137472, retained: 15600267264 2021-09-13 16:50:52 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 16:50:53 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 16:50:56 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13T16:51:09.642-07:00 [Info] memstats {"Alloc":395500232, "TotalAlloc":3536959447336, "Sys":976283896, "Lookups":0, "Mallocs":225932016,"Frees":225904884, "HeapAlloc":395500232, "HeapSys":934903808, "HeapIdle":536371200, "HeapInuse":398532608,"HeapReleased":433815552, "HeapObjects":27132,"GCSys":30965760, "LastGC":1631577069568592469,"PauseTotalNs":146111248403, "PauseNs":[568915, 758122, 669865, 336038, 1040131, 469723, 487289, 492138, 376948, 428922, 530410, 341143, 423604, 423857, 1312805, 777514, 868321, 1360486, 469135, 553867, 1025807, 1995815, 1327450, 397553, 979786, 365062, 1381755, 916299, 631702, 940424, 391924, 508174, 482203, 915992, 848729, 366124, 467179, 442831], "NumGC":18613} Allocated: 3726395304, active: 3727740928, metadata: 47935152 (n_thp 0), resident: 3802337280, mapped: 3818868736, retained: 15617536000 Mon Sep 13 16:51:15 PDT 2021 RSS = 3921832 2021-09-13 16:51:19 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T16:51:39.696-07:00 [Info] memstats {"Alloc":444490032, "TotalAlloc":3545010144464, "Sys":976283896, "Lookups":0, "Mallocs":226588192,"Frees":226561768, "HeapAlloc":444490032, "HeapSys":934871040, "HeapIdle":487309312, "HeapInuse":447561728,"HeapReleased":433324032, "HeapObjects":26424,"GCSys":30998528, "LastGC":1631577099683402752,"PauseTotalNs":146139301552, "PauseNs":[353575, 465627, 1033062, 755382, 467102, 374124, 422050, 2095796, 462140, 748894, 463221, 434276, 784418, 367826, 510208, 518266, 413751, 1046194, 630757, 661584, 397804, 412635, 925448, 6527000, 399088, 442617, 430266, 506309, 479637, 879661, 563258, 479867, 406149, 1259402, 385418, 550337], "NumGC":18649} Allocated: 3723347168, active: 3724677120, metadata: 47935152 (n_thp 0), resident: 3852230656, mapped: 3868762112, retained: 15567642624 2021-09-13T16:52:09.733-07:00 [Info] memstats {"Alloc":304123248, "TotalAlloc":3552964005240, "Sys":976283896, "Lookups":0, "Mallocs":227236534,"Frees":227215777, "HeapAlloc":304123248, "HeapSys":934969344, "HeapIdle":627294208, "HeapInuse":307675136,"HeapReleased":433594368, "HeapObjects":20757,"GCSys":30900224, "LastGC":1631577129420885164,"PauseTotalNs":146174601324, "PauseNs":[543704, 1254216, 1580979, 412343, 1827015, 752815, 489311, 485810, 420079, 711524, 499364, 469159, 514045, 402220, 530964, 752470, 781282, 549471, 416077, 396322, 381917, 775928, 1502715, 9249842, 860121, 3640377, 396678, 694634, 353325, 609908, 574641, 423120, 494398, 704011, 848987], "NumGC":18684} Allocated: 3722135840, active: 3723464704, metadata: 47935152 (n_thp 0), resident: 3818803200, mapped: 3835334656, retained: 15601070080 Mon Sep 13 16:52:15 PDT 2021 RSS = 3912912 2021-09-13T16:52:39.766-07:00 [Info] memstats {"Alloc":432741936, "TotalAlloc":3560739517672, "Sys":976283896, "Lookups":0, "Mallocs":227873881,"Frees":227848114, "HeapAlloc":432741936, "HeapSys":934838272, "HeapIdle":499023872, "HeapInuse":435814400,"HeapReleased":434896896, "HeapObjects":25767,"GCSys":31031296, "LastGC":1631577159745839953,"PauseTotalNs":146200777599, "PauseNs":[923813, 990735, 646922, 872253, 489772, 451086, 1650389, 1001568, 499924, 725711, 1439466, 525260, 638306, 362514, 484574, 916429, 365640, 1146276, 900362, 939050, 1374392, 507755, 925913, 395247, 374012, 888857, 336196, 419038, 442732, 1062088, 434675, 368461, 364819, 1161235, 1150805], "NumGC":18719} Allocated: 3731566416, active: 3732901888, metadata: 47935152 (n_thp 0), resident: 3819458560, mapped: 3835990016, retained: 15600414720 2021-09-13T16:53:09.836-07:00 [Info] memstats {"Alloc":444451264, "TotalAlloc":3568761569912, "Sys":976283896, "Lookups":0, "Mallocs":228528478,"Frees":228496756, "HeapAlloc":444451264, "HeapSys":934969344, "HeapIdle":487620608, "HeapInuse":447348736,"HeapReleased":434896896, "HeapObjects":31722,"GCSys":30900224, "LastGC":1631577189098317715,"PauseTotalNs":146230891520, "PauseNs":[328489, 1401345, 473777, 462359, 981031, 977116, 1049104, 541869, 1645308, 657697, 381085, 432543, 888653, 526208, 1116004, 4709612, 1808480, 414710, 521978, 413460, 658297, 606547, 1065087, 452946, 474736, 514330, 464813, 922306, 404778, 418796, 1196530, 1028263, 440715, 391195, 1343754], "NumGC":18754} Allocated: 3741981624, active: 3743322112, metadata: 47935152 (n_thp 0), resident: 3856027648, mapped: 3873869824, retained: 15562534912 Mon Sep 13 16:53:15 PDT 2021 RSS = 3941160 2021-09-13T16:53:39.875-07:00 [Info] memstats {"Alloc":373043320, "TotalAlloc":3576559889736, "Sys":976283896, "Lookups":0, "Mallocs":229168611,"Frees":229142322, "HeapAlloc":373043320, "HeapSys":934969344, "HeapIdle":558678016, "HeapInuse":376291328,"HeapReleased":434896896, "HeapObjects":26289,"GCSys":30900224, "LastGC":1631577219313912290,"PauseTotalNs":146260217370, "PauseNs":[1819489, 2577198, 1243305, 341376, 1225202, 474084, 457875, 437469, 479749, 490639, 1356780, 463573, 2022896, 934103, 441216, 1272041, 518745, 1098343, 573809, 759832, 688994, 709308, 944369, 454757, 497025, 1009871, 584099, 407760, 495538, 845381, 1103611, 430599, 1111101, 676874, 378839], "NumGC":18789} Allocated: 3743071304, active: 3744436224, metadata: 47935152 (n_thp 0), resident: 3844689920, mapped: 3861221376, retained: 15575183360 2021-09-13T16:54:09.911-07:00 [Info] memstats {"Alloc":406868848, "TotalAlloc":3584406136976, "Sys":976283896, "Lookups":0, "Mallocs":229809391,"Frees":229780155, "HeapAlloc":406868848, "HeapSys":934969344, "HeapIdle":525033472, "HeapInuse":409935872,"HeapReleased":434896896, "HeapObjects":29236,"GCSys":30900224, "LastGC":1631577249203182258,"PauseTotalNs":146285598660, "PauseNs":[416811, 418653, 1070782, 406334, 421636, 497421, 407694, 370255, 668942, 565996, 458447, 1006491, 860004, 457755, 1451168, 811926, 427236, 384484, 919437, 762986, 629045, 945919, 1605023, 796971, 967589, 438718, 562040, 520525, 1755536, 1767520, 404849, 441081, 944495, 490882, 326639], "NumGC":18824} Allocated: 3749209888, active: 3750596608, metadata: 47935152 (n_thp 0), resident: 3825127424, mapped: 3841658880, retained: 15594745856 Mon Sep 13 16:54:15 PDT 2021 RSS = 3938096 2021-09-13T16:54:39.955-07:00 [Info] memstats {"Alloc":394583856, "TotalAlloc":3592136583048, "Sys":976283896, "Lookups":0, "Mallocs":230441492,"Frees":230415256, "HeapAlloc":394583856, "HeapSys":934871040, "HeapIdle":537247744, "HeapInuse":397623296,"HeapReleased":434896896, "HeapObjects":26236,"GCSys":30998528, "LastGC":1631577279889478501,"PauseTotalNs":146312580468, "PauseNs":[531811, 1254611, 424445, 1992264, 942354, 676638, 441780, 588899, 1011298, 441226, 1516443, 720073, 777602, 666931, 979849, 380086, 394434, 794953, 486183, 1170721, 865965, 366751, 581957, 822539, 871367, 506596, 1156591, 1493650, 423954, 354630, 641024, 495926, 541795, 398592, 1267870], "NumGC":18859} Allocated: 3752125968, active: 3753512960, metadata: 47935152 (n_thp 0), resident: 3850162176, mapped: 3866693632, retained: 15569711104 2021-09-13T16:55:09.981-07:00 [Info] memstats {"Alloc":458823872, "TotalAlloc":3600002503808, "Sys":976283896, "Lookups":0, "Mallocs":231085911,"Frees":231052509, "HeapAlloc":458823872, "HeapSys":934969344, "HeapIdle":473358336, "HeapInuse":461611008,"HeapReleased":429965312, "HeapObjects":33402,"GCSys":30900224, "LastGC":1631577309168401497,"PauseTotalNs":146339308103, "PauseNs":[484711, 1173152, 997454, 400794, 490893, 1101505, 1467830, 760567, 530761, 564378, 848054, 690785, 704563, 1055229, 503462, 447739, 1223066, 489144, 480480, 913083, 336926, 861026, 717301, 880674, 1546571, 584900, 527859, 472383, 570001, 2189316, 954570, 540804, 578616, 639038], "NumGC":18893} Allocated: 3753155832, active: 3754557440, metadata: 47935152 (n_thp 0), resident: 3863019520, mapped: 3879550976, retained: 15556853760 Mon Sep 13 16:55:16 PDT 2021 RSS = 3946496 2021-09-13 16:55:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 242, "num_active_sctxs": 87, "num_free_sctxs": 155, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.56600, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3867004928, "memory_jemalloc_allocated": 3750349656, "memory_in_use": 3867004928, "memory_stats_size_total": 3301290557, "memory_stats_size_global": 1351202984, "memory_stats_size_page": 1217758536, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 137712, "reclaim_pending": 1573488, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16534, "total_ops": 1066396, "compacts": 3020609, "purges": 1522, "lss_data_size": 172364095, "lss_used_space": 2355571844, "lss_fragmentation": 92, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 218284490752, "bytes_incoming": 66648288728, "write_amp_avg": 2.30, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16534, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 45847700, "recovery_used_space": 469149315, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123284049920, "recovery_lss_frag": 90, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.94904 }2021-09-13T16:55:40.042-07:00 [Info] memstats {"Alloc":314715872, "TotalAlloc":3607658990968, "Sys":976283896, "Lookups":0, "Mallocs":231711124,"Frees":231687324, "HeapAlloc":314715872, "HeapSys":934903808, "HeapIdle":616972288, "HeapInuse":317931520,"HeapReleased":435134464, "HeapObjects":23800,"GCSys":30965760, "LastGC":1631577339831455263,"PauseTotalNs":146365536257, "PauseNs":[1029127, 432313, 553304, 414099, 507812, 485601, 484748, 520235, 755549, 976925, 1304214, 997435, 355476, 338775, 923226, 1357493, 1951423, 1440825, 1146328, 437057, 1066380, 489250, 1100177, 1091680, 445775, 474767, 334796, 478296, 408967, 1323293, 906682, 466282, 371012, 435306, 423526], "NumGC":18928} Allocated: 3752904648, active: 3754283008, metadata: 47935152 (n_thp 0), resident: 3847376896, mapped: 3863908352, retained: 15572496384 2021-09-13T16:56:10.096-07:00 [Info] memstats {"Alloc":361885008, "TotalAlloc":3615597882040, "Sys":976283896, "Lookups":0, "Mallocs":232357717,"Frees":232332464, "HeapAlloc":361885008, "HeapSys":934969344, "HeapIdle":569810944, "HeapInuse":365158400,"HeapReleased":430383104, "HeapObjects":25253,"GCSys":30900224, "LastGC":1631577369510020860,"PauseTotalNs":146393033508, "PauseNs":[324790, 452603, 351153, 526790, 1420811, 769516, 2065807, 439511, 855987, 474650, 1118601, 605646, 768600, 1495500, 405779, 1320753, 638576, 586664, 1394042, 393268, 359549, 513016, 626800, 1180907, 1179552, 370872, 573760, 1210204, 748793, 501289, 517116, 678311, 462776, 1769166, 396093], "NumGC":18963} Allocated: 3764354408, active: 3765706752, metadata: 47935152 (n_thp 0), resident: 3852476416, mapped: 3869663232, retained: 15566741504 Mon Sep 13 16:56:16 PDT 2021 RSS = 3947384 2021-09-13T16:56:40.148-07:00 [Info] memstats {"Alloc":389592600, "TotalAlloc":3623341935680, "Sys":976283896, "Lookups":0, "Mallocs":232989631,"Frees":232963123, "HeapAlloc":389592600, "HeapSys":934903808, "HeapIdle":542277632, "HeapInuse":392626176,"HeapReleased":434110464, "HeapObjects":26508,"GCSys":30965760, "LastGC":1631577400065807805,"PauseTotalNs":146416234606, "PauseNs":[946428, 801082, 666635, 530792, 321665, 1231267, 471573, 499799, 453681, 652631, 571339, 2908295, 399941, 782596, 533355, 884649, 383404, 380508, 376874, 674201, 455180, 917049, 439759, 403353, 594151, 771896, 487233, 491865, 1198558, 408656, 417902, 503182, 558488, 551404, 531707], "NumGC":18998} Allocated: 3752822584, active: 3754188800, metadata: 47935152 (n_thp 0), resident: 3839172608, mapped: 3855704064, retained: 15580700672 2021-09-13T16:57:10.187-07:00 [Info] memstats {"Alloc":311061296, "TotalAlloc":3631106273056, "Sys":976283896, "Lookups":0, "Mallocs":233622446,"Frees":233596102, "HeapAlloc":311061296, "HeapSys":934871040, "HeapIdle":620593152, "HeapInuse":314277888,"HeapReleased":434487296, "HeapObjects":26344,"GCSys":30998528, "LastGC":1631577429929988317,"PauseTotalNs":146456744360, "PauseNs":[397460, 385987, 1188486, 438189, 1112464, 1128316, 356536, 540854, 460852, 511362, 638379, 462329, 558222, 409844, 946285, 448200, 466537, 1422262, 413857, 384309, 452730, 812985, 876268, 592867, 608430, 17667915, 930493, 395061, 860245, 1065233, 384303, 1723958, 1052520, 416016], "NumGC":19032} Allocated: 3757536000, active: 3758907392, metadata: 47935152 (n_thp 0), resident: 3836026880, mapped: 3852558336, retained: 15583846400 Mon Sep 13 16:57:16 PDT 2021 RSS = 3952924 2021-09-13T16:57:40.227-07:00 [Info] memstats {"Alloc":397143432, "TotalAlloc":3639039956576, "Sys":976283896, "Lookups":0, "Mallocs":234267656,"Frees":234239451, "HeapAlloc":397143432, "HeapSys":934969344, "HeapIdle":534700032, "HeapInuse":400269312,"HeapReleased":434487296, "HeapObjects":28205,"GCSys":30900224, "LastGC":1631577459680242020,"PauseTotalNs":146483963840, "PauseNs":[775976, 535756, 584232, 588538, 463733, 394375, 1644126, 906570, 407011, 468545, 377609, 515934, 657968, 1145197, 524611, 1353495, 1110928, 1649284, 1188761, 1764098, 880806, 611636, 1017165, 1138697, 438705, 1122765, 632312, 371544, 1336997, 360428, 399365, 478797, 621470, 437453, 314593], "NumGC":19067} Allocated: 3753861120, active: 3755237376, metadata: 47935152 (n_thp 0), resident: 3859947520, mapped: 3876478976, retained: 15559925760 2021-09-13T16:58:10.267-07:00 [Info] memstats {"Alloc":411850264, "TotalAlloc":3646705954032, "Sys":976283896, "Lookups":0, "Mallocs":234894034,"Frees":234864676, "HeapAlloc":411850264, "HeapSys":934969344, "HeapIdle":520052736, "HeapInuse":414916608,"HeapReleased":432291840, "HeapObjects":29358,"GCSys":30900224, "LastGC":1631577489639478815,"PauseTotalNs":146510242797, "PauseNs":[465560, 418611, 620703, 841515, 410126, 734125, 576424, 1149029, 603433, 652018, 960165, 1028771, 368613, 738663, 587340, 382555, 453462, 1162185, 385537, 1234662, 1101672, 685484, 495697, 441556, 1006705, 1943116, 649316, 588139, 389130, 1192293, 1879034, 1120301, 568914, 444103], "NumGC":19101} Allocated: 3754278104, active: 3755655168, metadata: 47935152 (n_thp 0), resident: 3840655360, mapped: 3857186816, retained: 15579217920 Mon Sep 13 16:58:16 PDT 2021 RSS = 3957436 2021-09-13T16:58:40.310-07:00 [Info] memstats {"Alloc":359444512, "TotalAlloc":3654476046504, "Sys":976283896, "Lookups":0, "Mallocs":235526787,"Frees":235501649, "HeapAlloc":359444512, "HeapSys":934969344, "HeapIdle":572203008, "HeapInuse":362766336,"HeapReleased":432553984, "HeapObjects":25138,"GCSys":30900224, "LastGC":1631577519818617369,"PauseTotalNs":146539355988, "PauseNs":[1139028, 333117, 1064065, 848894, 1541896, 1032691, 978201, 377431, 1209102, 467531, 1105284, 970535, 1181540, 414688, 345850, 453407, 530260, 1185946, 591546, 813663, 343153, 533108, 1617387, 771935, 842489, 1099555, 591283, 395851, 461033, 1993311, 450541, 383154, 613753, 524064, 1907899], "NumGC":19136} Allocated: 3752703224, active: 3754082304, metadata: 47935152 (n_thp 0), resident: 3842228224, mapped: 3858759680, retained: 15577645056 2021-09-13T16:59:10.358-07:00 [Info] memstats {"Alloc":415174912, "TotalAlloc":3662113377376, "Sys":976283896, "Lookups":0, "Mallocs":236151889,"Frees":236122242, "HeapAlloc":415174912, "HeapSys":934969344, "HeapIdle":516734976, "HeapInuse":418234368,"HeapReleased":433455104, "HeapObjects":29647,"GCSys":30900224, "LastGC":1631577549619182497,"PauseTotalNs":146569767874, "PauseNs":[688727, 537308, 919037, 989993, 1179769, 719010, 789452, 2037518, 1128800, 371071, 800946, 1201409, 460867, 613610, 3877497, 500006, 1138775, 469462, 438519, 534179, 565856, 911789, 1203383, 557657, 525277, 775646, 930422, 628937, 433740, 881114, 415039, 1449438, 780669, 956964], "NumGC":19170} Allocated: 3759895072, active: 3761274880, metadata: 47935152 (n_thp 0), resident: 3843260416, mapped: 3859791872, retained: 15576612864 Mon Sep 13 16:59:16 PDT 2021 RSS = 3959864 2021-09-13T16:59:40.393-07:00 [Info] memstats {"Alloc":355790152, "TotalAlloc":3670164560848, "Sys":976283896, "Lookups":0, "Mallocs":236812684,"Frees":236787837, "HeapAlloc":355790152, "HeapSys":934969344, "HeapIdle":575832064, "HeapInuse":359137280,"HeapReleased":433455104, "HeapObjects":24847,"GCSys":30900224, "LastGC":1631577579888782236,"PauseTotalNs":146603466641, "PauseNs":[1124945, 507076, 1154858, 560180, 559385, 553025, 502845, 427523, 1541169, 409418, 9134764, 323470, 718291, 555241, 1006351, 364151, 1142411, 468895, 1756461, 1413579, 506441, 363389, 471620, 370775, 509980, 1243425, 436488, 503292, 500985, 1158137, 419290, 744384, 536169, 380789, 418090, 911475], "NumGC":19206} Allocated: 3754136824, active: 3755524096, metadata: 47935152 (n_thp 0), resident: 3847208960, mapped: 3863740416, retained: 15572664320 2021-09-13T17:00:10.431-07:00 [Info] memstats {"Alloc":315968768, "TotalAlloc":3678180262616, "Sys":976283896, "Lookups":0, "Mallocs":237468315,"Frees":237444137, "HeapAlloc":315968768, "HeapSys":934903808, "HeapIdle":615702528, "HeapInuse":319201280,"HeapReleased":434708480, "HeapObjects":24178,"GCSys":30965760, "LastGC":1631577610223148157,"PauseTotalNs":146627708490, "PauseNs":[918659, 448282, 805184, 342780, 962433, 373433, 478062, 574331, 448830, 1864533, 1138878, 439179, 439962, 533334, 389171, 858721, 452678, 951877, 593504, 366991, 521406, 466112, 1101838, 563183, 386342, 343399, 911655, 1130204, 365577, 459252, 1459610, 469540, 1224618, 528636, 429711, 499944], "NumGC":19242} Allocated: 3763327512, active: 3764731904, metadata: 47935152 (n_thp 0), resident: 3847651328, mapped: 3864182784, retained: 15572221952 Mon Sep 13 17:00:16 PDT 2021 RSS = 3953004 2021-09-13 17:00:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 242, "num_active_sctxs": 88, "num_free_sctxs": 155, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 236, "num_active_wctxs": 58, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.41800, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3848830976, "memory_jemalloc_allocated": 3757789768, "memory_in_use": 3848830976, "memory_stats_size_total": 3307977253, "memory_stats_size_global": 1357234072, "memory_stats_size_page": 1223920736, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 137648, "reclaim_pending": 2097984, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 1, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16628, "total_ops": 1090470, "compacts": 3144449, "purges": 1522, "lss_data_size": 172364095, "lss_used_space": 2355571844, "lss_fragmentation": 92, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 218284490752, "bytes_incoming": 66648288728, "write_amp_avg": 2.30, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16628, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 45847700, "recovery_used_space": 469149315, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123284049920, "recovery_lss_frag": 90, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.99423 }2021-09-13 17:00:25 [gsi-test] [0] Rollback() - PREPARE... 2021-09-13 17:00:25 [gsi-test] [0] Rollback() - DO!!! 2021-09-13 17:00:29 [gsi-test] [7] Rollback() - PREPARE... 2021-09-13 17:00:29 [gsi-test] [7] Rollback() - DO!!! 2021-09-13T17:00:40.471-07:00 [Info] memstats {"Alloc":394235904, "TotalAlloc":3681634320568, "Sys":976283896, "Lookups":0, "Mallocs":237768017,"Frees":237734944, "HeapAlloc":394235904, "HeapSys":934969344, "HeapIdle":537608192, "HeapInuse":397361152,"HeapReleased":434708480, "HeapObjects":33073,"GCSys":30900224, "LastGC":1631577630130046868,"PauseTotalNs":146639434648, "PauseNs":[822230, 615529, 1065006, 213517, 1294291, 815998, 636824, 537010, 521722, 992145, 474643, 1433145, 1041374, 852351, 410373], "NumGC":19257} Allocated: 3738812088, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3837952000, mapped: 3854483456, retained: 15581921280 2021-09-13 17:00:53 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 17:00:55 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 17:00:58 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13T17:01:10.502-07:00 [Info] memstats {"Alloc":397357280, "TotalAlloc":3681637441944, "Sys":976283896, "Lookups":0, "Mallocs":237791714,"Frees":237735996, "HeapAlloc":397357280, "HeapSys":934969344, "HeapIdle":534650880, "HeapInuse":400318464,"HeapReleased":434249728, "HeapObjects":55718,"GCSys":30900224, "LastGC":1631577630130046868,"PauseTotalNs":146639434648, "PauseNs":[], "NumGC":19257} Allocated: 3738822184, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3853156352, mapped: 3869687808, retained: 15566716928 Mon Sep 13 17:01:16 PDT 2021 RSS = 3947780 2021-09-13 17:01:20 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T17:01:40.533-07:00 [Info] memstats {"Alloc":399487784, "TotalAlloc":3681639572448, "Sys":976283896, "Lookups":0, "Mallocs":237813314,"Frees":237736635, "HeapAlloc":399487784, "HeapSys":934969344, "HeapIdle":532627456, "HeapInuse":402341888,"HeapReleased":433209344, "HeapObjects":76679,"GCSys":30900224, "LastGC":1631577630130046868,"PauseTotalNs":146639434648, "PauseNs":[], "NumGC":19257} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:02:10.564-07:00 [Info] memstats {"Alloc":401080440, "TotalAlloc":3681641165104, "Sys":976283896, "Lookups":0, "Mallocs":237832246,"Frees":237736644, "HeapAlloc":401080440, "HeapSys":934969344, "HeapIdle":530857984, "HeapInuse":404111360,"HeapReleased":432553984, "HeapObjects":95602,"GCSys":30900224, "LastGC":1631577630130046868,"PauseTotalNs":146639434648, "PauseNs":[], "NumGC":19257} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:02:16 PDT 2021 RSS = 3952564 2021-09-13T17:02:40.591-07:00 [Info] memstats {"Alloc":228238296, "TotalAlloc":3681642746232, "Sys":976283896, "Lookups":0, "Mallocs":237851214,"Frees":237819045, "HeapAlloc":228238296, "HeapSys":934969344, "HeapIdle":700317696, "HeapInuse":234651648,"HeapReleased":433594368, "HeapObjects":32169,"GCSys":30900224, "LastGC":1631577750136890974,"PauseTotalNs":146639995699, "PauseNs":[561051], "NumGC":19258} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:03:10.618-07:00 [Info] memstats {"Alloc":229823456, "TotalAlloc":3681644331392, "Sys":976283896, "Lookups":0, "Mallocs":237870155,"Frees":237819054, "HeapAlloc":229823456, "HeapSys":934969344, "HeapIdle":699809792, "HeapInuse":235159552,"HeapReleased":433594368, "HeapObjects":51101,"GCSys":30900224, "LastGC":1631577750136890974,"PauseTotalNs":146639995699, "PauseNs":[], "NumGC":19258} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:03:16 PDT 2021 RSS = 3952828 2021-09-13T17:03:40.646-07:00 [Info] memstats {"Alloc":231430280, "TotalAlloc":3681645938216, "Sys":976283896, "Lookups":0, "Mallocs":237889110,"Frees":237819063, "HeapAlloc":231430280, "HeapSys":934969344, "HeapIdle":699088896, "HeapInuse":235880448,"HeapReleased":433594368, "HeapObjects":70047,"GCSys":30900224, "LastGC":1631577750136890974,"PauseTotalNs":146639995699, "PauseNs":[], "NumGC":19258} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:04:10.677-07:00 [Info] memstats {"Alloc":233013528, "TotalAlloc":3681647521464, "Sys":976283896, "Lookups":0, "Mallocs":237908044,"Frees":237819072, "HeapAlloc":233013528, "HeapSys":934969344, "HeapIdle":698179584, "HeapInuse":236789760,"HeapReleased":433594368, "HeapObjects":88972,"GCSys":30900224, "LastGC":1631577750136890974,"PauseTotalNs":146639995699, "PauseNs":[], "NumGC":19258} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:04:16 PDT 2021 RSS = 3952828 2021-09-13T17:04:40.704-07:00 [Info] memstats {"Alloc":229166744, "TotalAlloc":3681649081056, "Sys":976283896, "Lookups":0, "Mallocs":237926992,"Frees":237883005, "HeapAlloc":229166744, "HeapSys":934969344, "HeapIdle":699457536, "HeapInuse":235511808,"HeapReleased":433594368, "HeapObjects":43987,"GCSys":30900224, "LastGC":1631577870154196633,"PauseTotalNs":146640320517, "PauseNs":[324818], "NumGC":19259} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:05:10.731-07:00 [Info] memstats {"Alloc":230744592, "TotalAlloc":3681650658904, "Sys":976283896, "Lookups":0, "Mallocs":237945938,"Frees":237883014, "HeapAlloc":230744592, "HeapSys":934969344, "HeapIdle":698785792, "HeapInuse":236183552,"HeapReleased":433594368, "HeapObjects":62924,"GCSys":30900224, "LastGC":1631577870154196633,"PauseTotalNs":146640320517, "PauseNs":[], "NumGC":19259} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:05:16 PDT 2021 RSS = 3952828 2021-09-13 17:05:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.21000, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 170482906, "lss_used_space": 2781682820, "lss_fragmentation": 93, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 218710601728, "bytes_incoming": 71478258656, "write_amp_avg": 1.98, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 45323172, "recovery_used_space": 580818563, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123395719168, "recovery_lss_frag": 92, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T17:05:40.758-07:00 [Info] memstats {"Alloc":232342504, "TotalAlloc":3681652256816, "Sys":976283896, "Lookups":0, "Mallocs":237965051,"Frees":237883059, "HeapAlloc":232342504, "HeapSys":934969344, "HeapIdle":698195968, "HeapInuse":236773376,"HeapReleased":433594368, "HeapObjects":81992,"GCSys":30900224, "LastGC":1631577870154196633,"PauseTotalNs":146640320517, "PauseNs":[], "NumGC":19259} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:06:10.790-07:00 [Info] memstats {"Alloc":233890984, "TotalAlloc":3681653805296, "Sys":976283896, "Lookups":0, "Mallocs":237983985,"Frees":237883068, "HeapAlloc":233890984, "HeapSys":934969344, "HeapIdle":697163776, "HeapInuse":237805568,"HeapReleased":433594368, "HeapObjects":100917,"GCSys":30900224, "LastGC":1631577870154196633,"PauseTotalNs":146640320517, "PauseNs":[], "NumGC":19259} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:06:16 PDT 2021 RSS = 3952828 2021-09-13T17:06:40.817-07:00 [Info] memstats {"Alloc":230114856, "TotalAlloc":3681655351016, "Sys":976283896, "Lookups":0, "Mallocs":238002948,"Frees":237947101, "HeapAlloc":230114856, "HeapSys":934969344, "HeapIdle":698433536, "HeapInuse":236535808,"HeapReleased":433594368, "HeapObjects":55847,"GCSys":30900224, "LastGC":1631577990167993686,"PauseTotalNs":146640710604, "PauseNs":[390087], "NumGC":19260} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:07:10.848-07:00 [Info] memstats {"Alloc":231696368, "TotalAlloc":3681656932528, "Sys":976283896, "Lookups":0, "Mallocs":238021896,"Frees":237947110, "HeapAlloc":231696368, "HeapSys":934969344, "HeapIdle":697810944, "HeapInuse":237158400,"HeapReleased":433594368, "HeapObjects":74786,"GCSys":30900224, "LastGC":1631577990167993686,"PauseTotalNs":146640710604, "PauseNs":[], "NumGC":19260} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:07:16 PDT 2021 RSS = 3952828 2021-09-13T17:07:40.877-07:00 [Info] memstats {"Alloc":233287944, "TotalAlloc":3681658524104, "Sys":976283896, "Lookups":0, "Mallocs":238040859,"Frees":237947119, "HeapAlloc":233287944, "HeapSys":934969344, "HeapIdle":697212928, "HeapInuse":237756416,"HeapReleased":433594368, "HeapObjects":93740,"GCSys":30900224, "LastGC":1631577990167993686,"PauseTotalNs":146640710604, "PauseNs":[], "NumGC":19260} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:08:10.905-07:00 [Info] memstats {"Alloc":234875624, "TotalAlloc":3681660111784, "Sys":976283896, "Lookups":0, "Mallocs":238059806,"Frees":237947128, "HeapAlloc":234875624, "HeapSys":934969344, "HeapIdle":696246272, "HeapInuse":238723072,"HeapReleased":433594368, "HeapObjects":112678,"GCSys":30900224, "LastGC":1631577990167993686,"PauseTotalNs":146640710604, "PauseNs":[], "NumGC":19260} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:08:16 PDT 2021 RSS = 3952828 2021-09-13T17:08:40.943-07:00 [Info] memstats {"Alloc":231064280, "TotalAlloc":3681661695664, "Sys":976283896, "Lookups":0, "Mallocs":238078774,"Frees":238011043, "HeapAlloc":231064280, "HeapSys":934969344, "HeapIdle":697540608, "HeapInuse":237428736,"HeapReleased":433594368, "HeapObjects":67731,"GCSys":30900224, "LastGC":1631578110194064054,"PauseTotalNs":146641144164, "PauseNs":[433560], "NumGC":19261} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:09:10.971-07:00 [Info] memstats {"Alloc":232644016, "TotalAlloc":3681663275400, "Sys":976283896, "Lookups":0, "Mallocs":238097715,"Frees":238011052, "HeapAlloc":232644016, "HeapSys":934969344, "HeapIdle":696901632, "HeapInuse":238067712,"HeapReleased":433594368, "HeapObjects":86663,"GCSys":30900224, "LastGC":1631578110194064054,"PauseTotalNs":146641144164, "PauseNs":[], "NumGC":19261} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:09:16 PDT 2021 RSS = 3952828 2021-09-13T17:09:40.998-07:00 [Info] memstats {"Alloc":234233656, "TotalAlloc":3681664865040, "Sys":976283896, "Lookups":0, "Mallocs":238116673,"Frees":238011061, "HeapAlloc":234233656, "HeapSys":934969344, "HeapIdle":696311808, "HeapInuse":238657536,"HeapReleased":433594368, "HeapObjects":105612,"GCSys":30900224, "LastGC":1631578110194064054,"PauseTotalNs":146641144164, "PauseNs":[], "NumGC":19261} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:10:11.030-07:00 [Info] memstats {"Alloc":235820472, "TotalAlloc":3681666451856, "Sys":976283896, "Lookups":0, "Mallocs":238135620,"Frees":238011070, "HeapAlloc":235820472, "HeapSys":934969344, "HeapIdle":695361536, "HeapInuse":239607808,"HeapReleased":433594368, "HeapObjects":124550,"GCSys":30900224, "LastGC":1631578110194064054,"PauseTotalNs":146641144164, "PauseNs":[], "NumGC":19261} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:10:17 PDT 2021 RSS = 3952828 2021-09-13 17:10:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.33800, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 170482906, "lss_used_space": 2781682820, "lss_fragmentation": 93, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 218710601728, "bytes_incoming": 71478258656, "write_amp_avg": 1.98, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 45323172, "recovery_used_space": 580818563, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123395719168, "recovery_lss_frag": 92, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T17:10:41.067-07:00 [Info] memstats {"Alloc":232044720, "TotalAlloc":3681670162408, "Sys":976283896, "Lookups":0, "Mallocs":238158544,"Frees":238078935, "HeapAlloc":232044720, "HeapSys":934969344, "HeapIdle":696500224, "HeapInuse":238469120,"HeapReleased":433594368, "HeapObjects":79609,"GCSys":30900224, "LastGC":1631578230215872046,"PauseTotalNs":146641688172, "PauseNs":[544008], "NumGC":19262} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13 17:10:54 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 17:10:56 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 17:10:58 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13T17:11:11.096-07:00 [Info] memstats {"Alloc":235079592, "TotalAlloc":3681673197280, "Sys":976283896, "Lookups":0, "Mallocs":238178579,"Frees":238078979, "HeapAlloc":235079592, "HeapSys":934969344, "HeapIdle":694542336, "HeapInuse":240427008,"HeapReleased":433594368, "HeapObjects":99600,"GCSys":30900224, "LastGC":1631578230215872046,"PauseTotalNs":146641688172, "PauseNs":[], "NumGC":19262} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:11:17 PDT 2021 RSS = 3952828 2021-09-13 17:11:22 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T17:11:41.123-07:00 [Info] memstats {"Alloc":237106640, "TotalAlloc":3681675224328, "Sys":976283896, "Lookups":0, "Mallocs":238197899,"Frees":238079000, "HeapAlloc":237106640, "HeapSys":934969344, "HeapIdle":693444608, "HeapInuse":241524736,"HeapReleased":433594368, "HeapObjects":118899,"GCSys":30900224, "LastGC":1631578230215872046,"PauseTotalNs":146641688172, "PauseNs":[], "NumGC":19262} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:12:11.162-07:00 [Info] memstats {"Alloc":238688256, "TotalAlloc":3681676805944, "Sys":976283896, "Lookups":0, "Mallocs":238216829,"Frees":238079009, "HeapAlloc":238688256, "HeapSys":934969344, "HeapIdle":692379648, "HeapInuse":242589696,"HeapReleased":433594368, "HeapObjects":137820,"GCSys":30900224, "LastGC":1631578230215872046,"PauseTotalNs":146641688172, "PauseNs":[], "NumGC":19262} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:12:17 PDT 2021 RSS = 3952828 2021-09-13T17:12:41.189-07:00 [Info] memstats {"Alloc":232984840, "TotalAlloc":3681678422768, "Sys":976283896, "Lookups":0, "Mallocs":238235787,"Frees":238144334, "HeapAlloc":232984840, "HeapSys":934969344, "HeapIdle":695533568, "HeapInuse":239435776,"HeapReleased":433594368, "HeapObjects":91453,"GCSys":30900224, "LastGC":1631578350239014572,"PauseTotalNs":146642137995, "PauseNs":[449823], "NumGC":19263} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:13:11.221-07:00 [Info] memstats {"Alloc":234613920, "TotalAlloc":3681680051848, "Sys":976283896, "Lookups":0, "Mallocs":238254740,"Frees":238144343, "HeapAlloc":234613920, "HeapSys":934969344, "HeapIdle":694878208, "HeapInuse":240091136,"HeapReleased":433594368, "HeapObjects":110397,"GCSys":30900224, "LastGC":1631578350239014572,"PauseTotalNs":146642137995, "PauseNs":[], "NumGC":19263} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:13:17 PDT 2021 RSS = 3952828 2021-09-13T17:13:41.248-07:00 [Info] memstats {"Alloc":236246872, "TotalAlloc":3681681684800, "Sys":976283896, "Lookups":0, "Mallocs":238273696,"Frees":238144352, "HeapAlloc":236246872, "HeapSys":934969344, "HeapIdle":694296576, "HeapInuse":240672768,"HeapReleased":433594368, "HeapObjects":129344,"GCSys":30900224, "LastGC":1631578350239014572,"PauseTotalNs":146642137995, "PauseNs":[], "NumGC":19263} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:14:11.275-07:00 [Info] memstats {"Alloc":237865320, "TotalAlloc":3681683303248, "Sys":976283896, "Lookups":0, "Mallocs":238292637,"Frees":238144361, "HeapAlloc":237865320, "HeapSys":934969344, "HeapIdle":693215232, "HeapInuse":241754112,"HeapReleased":433594368, "HeapObjects":148276,"GCSys":30900224, "LastGC":1631578350239014572,"PauseTotalNs":146642137995, "PauseNs":[], "NumGC":19263} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:14:17 PDT 2021 RSS = 3952828 2021-09-13T17:14:41.304-07:00 [Info] memstats {"Alloc":233919448, "TotalAlloc":3681684918232, "Sys":976283896, "Lookups":0, "Mallocs":238311610,"Frees":238208312, "HeapAlloc":233919448, "HeapSys":934969344, "HeapIdle":694706176, "HeapInuse":240263168,"HeapReleased":433594368, "HeapObjects":103298,"GCSys":30900224, "LastGC":1631578470261736526,"PauseTotalNs":146642550126, "PauseNs":[412131], "NumGC":19264} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:15:11.332-07:00 [Info] memstats {"Alloc":235541952, "TotalAlloc":3681686540736, "Sys":976283896, "Lookups":0, "Mallocs":238330553,"Frees":238208321, "HeapAlloc":235541952, "HeapSys":934969344, "HeapIdle":694083584, "HeapInuse":240885760,"HeapReleased":433594368, "HeapObjects":122232,"GCSys":30900224, "LastGC":1631578470261736526,"PauseTotalNs":146642550126, "PauseNs":[], "NumGC":19264} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:15:17 PDT 2021 RSS = 3952828 2021-09-13 17:15:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.24200, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 170482906, "lss_used_space": 2781682820, "lss_fragmentation": 93, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 218710601728, "bytes_incoming": 71478258656, "write_amp_avg": 1.98, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 45323172, "recovery_used_space": 580818563, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123395719168, "recovery_lss_frag": 92, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T17:15:41.360-07:00 [Info] memstats {"Alloc":237287456, "TotalAlloc":3681688286240, "Sys":976283896, "Lookups":0, "Mallocs":238349916,"Frees":238208381, "HeapAlloc":237287456, "HeapSys":934969344, "HeapIdle":693231616, "HeapInuse":241737728,"HeapReleased":433594368, "HeapObjects":141535,"GCSys":30900224, "LastGC":1631578470261736526,"PauseTotalNs":146642550126, "PauseNs":[], "NumGC":19264} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:16:11.388-07:00 [Info] memstats {"Alloc":238914432, "TotalAlloc":3681689913216, "Sys":976283896, "Lookups":0, "Mallocs":238368864,"Frees":238208390, "HeapAlloc":238914432, "HeapSys":934969344, "HeapIdle":692248576, "HeapInuse":242720768,"HeapReleased":433594368, "HeapObjects":160474,"GCSys":30900224, "LastGC":1631578470261736526,"PauseTotalNs":146642550126, "PauseNs":[], "NumGC":19264} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:16:17 PDT 2021 RSS = 3952828 2021-09-13T17:16:41.414-07:00 [Info] memstats {"Alloc":234883032, "TotalAlloc":3681691542072, "Sys":976283896, "Lookups":0, "Mallocs":238387829,"Frees":238272675, "HeapAlloc":234883032, "HeapSys":934969344, "HeapIdle":693690368, "HeapInuse":241278976,"HeapReleased":433594368, "HeapObjects":115154,"GCSys":30900224, "LastGC":1631578590303091165,"PauseTotalNs":146643114140, "PauseNs":[564014], "NumGC":19265} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:17:11.444-07:00 [Info] memstats {"Alloc":236511168, "TotalAlloc":3681693170208, "Sys":976283896, "Lookups":0, "Mallocs":238406772,"Frees":238272684, "HeapAlloc":236511168, "HeapSys":934969344, "HeapIdle":693043200, "HeapInuse":241926144,"HeapReleased":433594368, "HeapObjects":134088,"GCSys":30900224, "LastGC":1631578590303091165,"PauseTotalNs":146643114140, "PauseNs":[], "NumGC":19265} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:17:17 PDT 2021 RSS = 3952828 2021-09-13T17:17:41.473-07:00 [Info] memstats {"Alloc":238128056, "TotalAlloc":3681694787096, "Sys":976283896, "Lookups":0, "Mallocs":238425729,"Frees":238272693, "HeapAlloc":238128056, "HeapSys":934969344, "HeapIdle":692387840, "HeapInuse":242581504,"HeapReleased":433594368, "HeapObjects":153036,"GCSys":30900224, "LastGC":1631578590303091165,"PauseTotalNs":146643114140, "PauseNs":[], "NumGC":19265} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:18:11.500-07:00 [Info] memstats {"Alloc":239742456, "TotalAlloc":3681696401496, "Sys":976283896, "Lookups":0, "Mallocs":238444671,"Frees":238272702, "HeapAlloc":239742456, "HeapSys":934969344, "HeapIdle":691363840, "HeapInuse":243605504,"HeapReleased":433594368, "HeapObjects":171969,"GCSys":30900224, "LastGC":1631578590303091165,"PauseTotalNs":146643114140, "PauseNs":[], "NumGC":19265} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:18:17 PDT 2021 RSS = 3952828 2021-09-13T17:18:41.532-07:00 [Info] memstats {"Alloc":235832392, "TotalAlloc":3681698053248, "Sys":976283896, "Lookups":0, "Mallocs":238463635,"Frees":238336624, "HeapAlloc":235832392, "HeapSys":934969344, "HeapIdle":692813824, "HeapInuse":242155520,"HeapReleased":433594368, "HeapObjects":127011,"GCSys":30900224, "LastGC":1631578710331409835,"PauseTotalNs":146643449872, "PauseNs":[335732], "NumGC":19266} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:19:11.561-07:00 [Info] memstats {"Alloc":237481936, "TotalAlloc":3681699702792, "Sys":976283896, "Lookups":0, "Mallocs":238482581,"Frees":238336633, "HeapAlloc":237481936, "HeapSys":934969344, "HeapIdle":692076544, "HeapInuse":242892800,"HeapReleased":433594368, "HeapObjects":145948,"GCSys":30900224, "LastGC":1631578710331409835,"PauseTotalNs":146643449872, "PauseNs":[], "NumGC":19266} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:19:17 PDT 2021 RSS = 3952828 2021-09-13T17:19:41.606-07:00 [Info] memstats {"Alloc":239126776, "TotalAlloc":3681701347632, "Sys":976283896, "Lookups":0, "Mallocs":238501550,"Frees":238336642, "HeapAlloc":239126776, "HeapSys":934969344, "HeapIdle":691396608, "HeapInuse":243572736,"HeapReleased":433594368, "HeapObjects":164908,"GCSys":30900224, "LastGC":1631578710331409835,"PauseTotalNs":146643449872, "PauseNs":[], "NumGC":19266} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:20:11.634-07:00 [Info] memstats {"Alloc":240762712, "TotalAlloc":3681702983568, "Sys":976283896, "Lookups":0, "Mallocs":238520491,"Frees":238336651, "HeapAlloc":240762712, "HeapSys":934969344, "HeapIdle":690339840, "HeapInuse":244629504,"HeapReleased":433594368, "HeapObjects":183840,"GCSys":30900224, "LastGC":1631578710331409835,"PauseTotalNs":146643449872, "PauseNs":[], "NumGC":19266} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:20:17 PDT 2021 RSS = 3952828 2021-09-13 17:20:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.24800, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 574063809, "lss_used_space": 3249925252, "lss_fragmentation": 82, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 219178844160, "bytes_incoming": 71502511416, "write_amp_avg": 2.00, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 153900612, "recovery_used_space": 706750083, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123521650688, "recovery_lss_frag": 78, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T17:20:41.662-07:00 [Info] memstats {"Alloc":236799280, "TotalAlloc":3681706735024, "Sys":976283896, "Lookups":0, "Mallocs":238543403,"Frees":238404525, "HeapAlloc":236799280, "HeapSys":934969344, "HeapIdle":691773440, "HeapInuse":243195904,"HeapReleased":433594368, "HeapObjects":138878,"GCSys":30900224, "LastGC":1631578830372583815,"PauseTotalNs":146643817340, "PauseNs":[367468], "NumGC":19267} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13 17:20:54 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 17:20:56 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 17:20:58 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13T17:21:11.701-07:00 [Info] memstats {"Alloc":239904264, "TotalAlloc":3681709840008, "Sys":976283896, "Lookups":0, "Mallocs":238563426,"Frees":238404569, "HeapAlloc":239904264, "HeapSys":934969344, "HeapIdle":689700864, "HeapInuse":245268480,"HeapReleased":433594368, "HeapObjects":158857,"GCSys":30900224, "LastGC":1631578830372583815,"PauseTotalNs":146643817340, "PauseNs":[], "NumGC":19267} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:21:17 PDT 2021 RSS = 3952828 2021-09-13 17:21:22 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T17:21:41.729-07:00 [Info] memstats {"Alloc":242005760, "TotalAlloc":3681711941504, "Sys":976283896, "Lookups":0, "Mallocs":238582743,"Frees":238404590, "HeapAlloc":242005760, "HeapSys":934969344, "HeapIdle":688537600, "HeapInuse":246431744,"HeapReleased":433594368, "HeapObjects":178153,"GCSys":30900224, "LastGC":1631578830372583815,"PauseTotalNs":146643817340, "PauseNs":[], "NumGC":19267} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:22:11.763-07:00 [Info] memstats {"Alloc":243619600, "TotalAlloc":3681713555344, "Sys":976283896, "Lookups":0, "Mallocs":238601689,"Frees":238404599, "HeapAlloc":243619600, "HeapSys":934969344, "HeapIdle":687456256, "HeapInuse":247513088,"HeapReleased":433594368, "HeapObjects":197090,"GCSys":30900224, "LastGC":1631578830372583815,"PauseTotalNs":146643817340, "PauseNs":[], "NumGC":19267} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:22:17 PDT 2021 RSS = 3952828 2021-09-13T17:22:41.790-07:00 [Info] memstats {"Alloc":237705496, "TotalAlloc":3681715156920, "Sys":976283896, "Lookups":0, "Mallocs":238620659,"Frees":238469933, "HeapAlloc":237705496, "HeapSys":934969344, "HeapIdle":690905088, "HeapInuse":244064256,"HeapReleased":433594368, "HeapObjects":150726,"GCSys":30900224, "LastGC":1631578950411279951,"PauseTotalNs":146644180973, "PauseNs":[363633], "NumGC":19268} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:23:11.822-07:00 [Info] memstats {"Alloc":239288304, "TotalAlloc":3681716739728, "Sys":976283896, "Lookups":0, "Mallocs":238639607,"Frees":238469942, "HeapAlloc":239288304, "HeapSys":934969344, "HeapIdle":690249728, "HeapInuse":244719616,"HeapReleased":433594368, "HeapObjects":169665,"GCSys":30900224, "LastGC":1631578950411279951,"PauseTotalNs":146644180973, "PauseNs":[], "NumGC":19268} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:23:17 PDT 2021 RSS = 3952828 2021-09-13T17:23:41.848-07:00 [Info] memstats {"Alloc":240878328, "TotalAlloc":3681718329752, "Sys":976283896, "Lookups":0, "Mallocs":238658566,"Frees":238469951, "HeapAlloc":240878328, "HeapSys":934969344, "HeapIdle":689651712, "HeapInuse":245317632,"HeapReleased":433594368, "HeapObjects":188615,"GCSys":30900224, "LastGC":1631578950411279951,"PauseTotalNs":146644180973, "PauseNs":[], "NumGC":19268} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:24:11.874-07:00 [Info] memstats {"Alloc":242457784, "TotalAlloc":3681719909208, "Sys":976283896, "Lookups":0, "Mallocs":238677512,"Frees":238469960, "HeapAlloc":242457784, "HeapSys":934969344, "HeapIdle":688709632, "HeapInuse":246259712,"HeapReleased":433594368, "HeapObjects":207552,"GCSys":30900224, "LastGC":1631578950411279951,"PauseTotalNs":146644180973, "PauseNs":[], "NumGC":19268} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:24:17 PDT 2021 RSS = 3952828 2021-09-13T17:24:41.899-07:00 [Info] memstats {"Alloc":238646776, "TotalAlloc":3681721490768, "Sys":976283896, "Lookups":0, "Mallocs":238696478,"Frees":238533920, "HeapAlloc":238646776, "HeapSys":934969344, "HeapIdle":689930240, "HeapInuse":245039104,"HeapReleased":433594368, "HeapObjects":162558,"GCSys":30900224, "LastGC":1631579070460855838,"PauseTotalNs":146644564758, "PauseNs":[383785], "NumGC":19269} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:25:11.926-07:00 [Info] memstats {"Alloc":240214224, "TotalAlloc":3681723058216, "Sys":976283896, "Lookups":0, "Mallocs":238715419,"Frees":238533929, "HeapAlloc":240214224, "HeapSys":934969344, "HeapIdle":689299456, "HeapInuse":245669888,"HeapReleased":433594368, "HeapObjects":181490,"GCSys":30900224, "LastGC":1631579070460855838,"PauseTotalNs":146644564758, "PauseNs":[], "NumGC":19269} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:25:18 PDT 2021 RSS = 3952828 2021-09-13 17:25:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.33833, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 574063809, "lss_used_space": 3249925252, "lss_fragmentation": 82, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 219178844160, "bytes_incoming": 71502511416, "write_amp_avg": 2.00, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 153900612, "recovery_used_space": 706750083, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123521650688, "recovery_lss_frag": 78, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T17:25:41.957-07:00 [Info] memstats {"Alloc":241815192, "TotalAlloc":3681724659184, "Sys":976283896, "Lookups":0, "Mallocs":238734545,"Frees":238533974, "HeapAlloc":241815192, "HeapSys":934969344, "HeapIdle":688668672, "HeapInuse":246300672,"HeapReleased":433594368, "HeapObjects":200571,"GCSys":30900224, "LastGC":1631579070460855838,"PauseTotalNs":146644564758, "PauseNs":[], "NumGC":19269} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:26:11.983-07:00 [Info] memstats {"Alloc":243347640, "TotalAlloc":3681726191632, "Sys":976283896, "Lookups":0, "Mallocs":238753497,"Frees":238533983, "HeapAlloc":243347640, "HeapSys":934969344, "HeapIdle":687677440, "HeapInuse":247291904,"HeapReleased":433594368, "HeapObjects":219514,"GCSys":30900224, "LastGC":1631579070460855838,"PauseTotalNs":146644564758, "PauseNs":[], "NumGC":19269} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:26:18 PDT 2021 RSS = 3952828 2021-09-13T17:26:42.011-07:00 [Info] memstats {"Alloc":239602648, "TotalAlloc":3681727759432, "Sys":976283896, "Lookups":0, "Mallocs":238772463,"Frees":238598066, "HeapAlloc":239602648, "HeapSys":934969344, "HeapIdle":688955392, "HeapInuse":246013952,"HeapReleased":433594368, "HeapObjects":174397,"GCSys":30900224, "LastGC":1631579190499033702,"PauseTotalNs":146644925986, "PauseNs":[361228], "NumGC":19270} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:27:12.045-07:00 [Info] memstats {"Alloc":241184960, "TotalAlloc":3681729341744, "Sys":976283896, "Lookups":0, "Mallocs":238791407,"Frees":238598075, "HeapAlloc":241184960, "HeapSys":934969344, "HeapIdle":688300032, "HeapInuse":246669312,"HeapReleased":433594368, "HeapObjects":193332,"GCSys":30900224, "LastGC":1631579190499033702,"PauseTotalNs":146644925986, "PauseNs":[], "NumGC":19270} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:27:18 PDT 2021 RSS = 3952828 2021-09-13T17:27:42.073-07:00 [Info] memstats {"Alloc":242769896, "TotalAlloc":3681730926680, "Sys":976283896, "Lookups":0, "Mallocs":238810374,"Frees":238598084, "HeapAlloc":242769896, "HeapSys":934969344, "HeapIdle":687693824, "HeapInuse":247275520,"HeapReleased":433594368, "HeapObjects":212290,"GCSys":30900224, "LastGC":1631579190499033702,"PauseTotalNs":146644925986, "PauseNs":[], "NumGC":19270} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:28:12.096-07:00 [Info] memstats {"Alloc":244331640, "TotalAlloc":3681732488424, "Sys":976283896, "Lookups":0, "Mallocs":238829322,"Frees":238598093, "HeapAlloc":244331640, "HeapSys":934969344, "HeapIdle":686776320, "HeapInuse":248193024,"HeapReleased":433594368, "HeapObjects":231229,"GCSys":30900224, "LastGC":1631579190499033702,"PauseTotalNs":146644925986, "PauseNs":[], "NumGC":19270} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:28:18 PDT 2021 RSS = 3952828 2021-09-13T17:28:42.122-07:00 [Info] memstats {"Alloc":240527352, "TotalAlloc":3681734028608, "Sys":976283896, "Lookups":0, "Mallocs":238848283,"Frees":238662049, "HeapAlloc":240527352, "HeapSys":934969344, "HeapIdle":687964160, "HeapInuse":247005184,"HeapReleased":433594368, "HeapObjects":186234,"GCSys":30900224, "LastGC":1631579310566067809,"PauseTotalNs":146645286094, "PauseNs":[360108], "NumGC":19271} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:29:12.153-07:00 [Info] memstats {"Alloc":242084896, "TotalAlloc":3681735586152, "Sys":976283896, "Lookups":0, "Mallocs":238867239,"Frees":238662058, "HeapAlloc":242084896, "HeapSys":934969344, "HeapIdle":687448064, "HeapInuse":247521280,"HeapReleased":433594368, "HeapObjects":205181,"GCSys":30900224, "LastGC":1631579310566067809,"PauseTotalNs":146645286094, "PauseNs":[], "NumGC":19271} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:29:18 PDT 2021 RSS = 3952828 2021-09-13T17:29:42.181-07:00 [Info] memstats {"Alloc":243657352, "TotalAlloc":3681737158608, "Sys":976283896, "Lookups":0, "Mallocs":238886208,"Frees":238662067, "HeapAlloc":243657352, "HeapSys":934969344, "HeapIdle":686850048, "HeapInuse":248119296,"HeapReleased":433594368, "HeapObjects":224141,"GCSys":30900224, "LastGC":1631579310566067809,"PauseTotalNs":146645286094, "PauseNs":[], "NumGC":19271} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:30:12.209-07:00 [Info] memstats {"Alloc":245239784, "TotalAlloc":3681738741040, "Sys":976283896, "Lookups":0, "Mallocs":238905145,"Frees":238662076, "HeapAlloc":245239784, "HeapSys":934969344, "HeapIdle":685842432, "HeapInuse":249126912,"HeapReleased":433594368, "HeapObjects":243069,"GCSys":30900224, "LastGC":1631579310566067809,"PauseTotalNs":146645286094, "PauseNs":[], "NumGC":19271} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:30:18 PDT 2021 RSS = 3952828 2021-09-13 17:30:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.28200, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 574063809, "lss_used_space": 3249925252, "lss_fragmentation": 82, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 219178844160, "bytes_incoming": 71502511416, "write_amp_avg": 2.00, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 153900612, "recovery_used_space": 706750083, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123521650688, "recovery_lss_frag": 78, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T17:30:42.236-07:00 [Info] memstats {"Alloc":241529488, "TotalAlloc":3681742542048, "Sys":976283896, "Lookups":0, "Mallocs":238928305,"Frees":238730200, "HeapAlloc":241529488, "HeapSys":934969344, "HeapIdle":686981120, "HeapInuse":247988224,"HeapReleased":433594368, "HeapObjects":198105,"GCSys":30900224, "LastGC":1631579430628855568,"PauseTotalNs":146645658154, "PauseNs":[372060], "NumGC":19272} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13 17:30:54 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 17:30:56 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 17:30:58 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13T17:31:12.267-07:00 [Info] memstats {"Alloc":244610344, "TotalAlloc":3681745622904, "Sys":976283896, "Lookups":0, "Mallocs":238948337,"Frees":238730244, "HeapAlloc":244610344, "HeapSys":934969344, "HeapIdle":684941312, "HeapInuse":250028032,"HeapReleased":433594368, "HeapObjects":218093,"GCSys":30900224, "LastGC":1631579430628855568,"PauseTotalNs":146645658154, "PauseNs":[], "NumGC":19272} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:31:18 PDT 2021 RSS = 3952828 2021-09-13 17:31:22 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T17:31:42.295-07:00 [Info] memstats {"Alloc":246688240, "TotalAlloc":3681747700800, "Sys":976283896, "Lookups":0, "Mallocs":238967647,"Frees":238730265, "HeapAlloc":246688240, "HeapSys":934969344, "HeapIdle":683843584, "HeapInuse":251125760,"HeapReleased":433594368, "HeapObjects":237382,"GCSys":30900224, "LastGC":1631579430628855568,"PauseTotalNs":146645658154, "PauseNs":[], "NumGC":19272} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:32:12.334-07:00 [Info] memstats {"Alloc":248304960, "TotalAlloc":3681749317520, "Sys":976283896, "Lookups":0, "Mallocs":238986591,"Frees":238730274, "HeapAlloc":248304960, "HeapSys":934969344, "HeapIdle":682803200, "HeapInuse":252166144,"HeapReleased":433594368, "HeapObjects":256317,"GCSys":30900224, "LastGC":1631579430628855568,"PauseTotalNs":146645658154, "PauseNs":[], "NumGC":19272} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:32:18 PDT 2021 RSS = 3952828 2021-09-13T17:32:42.364-07:00 [Info] memstats {"Alloc":242446168, "TotalAlloc":3681750930616, "Sys":976283896, "Lookups":0, "Mallocs":239005544,"Frees":238795616, "HeapAlloc":242446168, "HeapSys":934969344, "HeapIdle":686104576, "HeapInuse":248864768,"HeapReleased":433594368, "HeapObjects":209928,"GCSys":30900224, "LastGC":1631579550684794185,"PauseTotalNs":146646101884, "PauseNs":[443730], "NumGC":19273} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:33:12.392-07:00 [Info] memstats {"Alloc":244041440, "TotalAlloc":3681752525888, "Sys":976283896, "Lookups":0, "Mallocs":239024470,"Frees":238795625, "HeapAlloc":244041440, "HeapSys":934969344, "HeapIdle":685498368, "HeapInuse":249470976,"HeapReleased":433594368, "HeapObjects":228845,"GCSys":30900224, "LastGC":1631579550684794185,"PauseTotalNs":146646101884, "PauseNs":[], "NumGC":19273} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:33:18 PDT 2021 RSS = 3952880 2021-09-13T17:33:42.420-07:00 [Info] memstats {"Alloc":245649368, "TotalAlloc":3681754133816, "Sys":976283896, "Lookups":0, "Mallocs":239043435,"Frees":238795634, "HeapAlloc":245649368, "HeapSys":934969344, "HeapIdle":684883968, "HeapInuse":250085376,"HeapReleased":433594368, "HeapObjects":247801,"GCSys":30900224, "LastGC":1631579550684794185,"PauseTotalNs":146646101884, "PauseNs":[], "NumGC":19273} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:34:12.448-07:00 [Info] memstats {"Alloc":247252472, "TotalAlloc":3681755736920, "Sys":976283896, "Lookups":0, "Mallocs":239062364,"Frees":238795643, "HeapAlloc":247252472, "HeapSys":934969344, "HeapIdle":683892736, "HeapInuse":251076608,"HeapReleased":433594368, "HeapObjects":266721,"GCSys":30900224, "LastGC":1631579550684794185,"PauseTotalNs":146646101884, "PauseNs":[], "NumGC":19273} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:34:18 PDT 2021 RSS = 3952880 2021-09-13T17:34:42.475-07:00 [Info] memstats {"Alloc":243391048, "TotalAlloc":3681757361136, "Sys":976283896, "Lookups":0, "Mallocs":239081329,"Frees":238859570, "HeapAlloc":243391048, "HeapSys":934969344, "HeapIdle":685178880, "HeapInuse":249790464,"HeapReleased":433594368, "HeapObjects":221759,"GCSys":30900224, "LastGC":1631579670743356251,"PauseTotalNs":146646464832, "PauseNs":[362948], "NumGC":19274} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:35:12.501-07:00 [Info] memstats {"Alloc":244980544, "TotalAlloc":3681758950632, "Sys":976283896, "Lookups":0, "Mallocs":239100280,"Frees":238859579, "HeapAlloc":244980544, "HeapSys":934969344, "HeapIdle":684621824, "HeapInuse":250347520,"HeapReleased":433594368, "HeapObjects":240701,"GCSys":30900224, "LastGC":1631579670743356251,"PauseTotalNs":146646464832, "PauseNs":[], "NumGC":19274} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:35:18 PDT 2021 RSS = 3952880 2021-09-13 17:35:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.30200, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 574063809, "lss_used_space": 3249990788, "lss_fragmentation": 82, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 219178909696, "bytes_incoming": 71502511416, "write_amp_avg": 2.00, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 153900612, "recovery_used_space": 706782851, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123521683456, "recovery_lss_frag": 78, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T17:35:42.528-07:00 [Info] memstats {"Alloc":246616120, "TotalAlloc":3681760586208, "Sys":976283896, "Lookups":0, "Mallocs":239119407,"Frees":238859624, "HeapAlloc":246616120, "HeapSys":934969344, "HeapIdle":683892736, "HeapInuse":251076608,"HeapReleased":433594368, "HeapObjects":259783,"GCSys":30900224, "LastGC":1631579670743356251,"PauseTotalNs":146646464832, "PauseNs":[], "NumGC":19274} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:36:12.563-07:00 [Info] memstats {"Alloc":248192168, "TotalAlloc":3681762162256, "Sys":976283896, "Lookups":0, "Mallocs":239138350,"Frees":238859633, "HeapAlloc":248192168, "HeapSys":934969344, "HeapIdle":682803200, "HeapInuse":252166144,"HeapReleased":433594368, "HeapObjects":278717,"GCSys":30900224, "LastGC":1631579670743356251,"PauseTotalNs":146646464832, "PauseNs":[], "NumGC":19274} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:36:18 PDT 2021 RSS = 3952880 2021-09-13T17:36:42.589-07:00 [Info] memstats {"Alloc":244337896, "TotalAlloc":3681763746632, "Sys":976283896, "Lookups":0, "Mallocs":239157316,"Frees":238923722, "HeapAlloc":244337896, "HeapSys":934969344, "HeapIdle":684179456, "HeapInuse":250789888,"HeapReleased":433594368, "HeapObjects":233594,"GCSys":30900224, "LastGC":1631579790816735630,"PauseTotalNs":146651224460, "PauseNs":[4759628], "NumGC":19275} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:37:12.624-07:00 [Info] memstats {"Alloc":245936720, "TotalAlloc":3681765345456, "Sys":976283896, "Lookups":0, "Mallocs":239176266,"Frees":238923731, "HeapAlloc":245936720, "HeapSys":934969344, "HeapIdle":683630592, "HeapInuse":251338752,"HeapReleased":433594368, "HeapObjects":252535,"GCSys":30900224, "LastGC":1631579790816735630,"PauseTotalNs":146651224460, "PauseNs":[], "NumGC":19275} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:37:18 PDT 2021 RSS = 3952880 2021-09-13T17:37:42.650-07:00 [Info] memstats {"Alloc":247522568, "TotalAlloc":3681766931304, "Sys":976283896, "Lookups":0, "Mallocs":239195224,"Frees":238923740, "HeapAlloc":247522568, "HeapSys":934969344, "HeapIdle":682950656, "HeapInuse":252018688,"HeapReleased":433594368, "HeapObjects":271484,"GCSys":30900224, "LastGC":1631579790816735630,"PauseTotalNs":146651224460, "PauseNs":[], "NumGC":19275} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:38:12.676-07:00 [Info] memstats {"Alloc":249108872, "TotalAlloc":3681768517608, "Sys":976283896, "Lookups":0, "Mallocs":239214174,"Frees":238923749, "HeapAlloc":249108872, "HeapSys":934969344, "HeapIdle":681967616, "HeapInuse":253001728,"HeapReleased":433594368, "HeapObjects":290425,"GCSys":30900224, "LastGC":1631579790816735630,"PauseTotalNs":146651224460, "PauseNs":[], "NumGC":19275} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:38:18 PDT 2021 RSS = 3952880 2021-09-13T17:38:42.703-07:00 [Info] memstats {"Alloc":245271944, "TotalAlloc":3681770113504, "Sys":976283896, "Lookups":0, "Mallocs":239233141,"Frees":238987718, "HeapAlloc":245271944, "HeapSys":934969344, "HeapIdle":683286528, "HeapInuse":251682816,"HeapReleased":433594368, "HeapObjects":245423,"GCSys":30900224, "LastGC":1631579910873318756,"PauseTotalNs":146651638427, "PauseNs":[413967], "NumGC":19276} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:39:12.733-07:00 [Info] memstats {"Alloc":246827728, "TotalAlloc":3681771669288, "Sys":976283896, "Lookups":0, "Mallocs":239252092,"Frees":238987727, "HeapAlloc":246827728, "HeapSys":934969344, "HeapIdle":682680320, "HeapInuse":252289024,"HeapReleased":433594368, "HeapObjects":264365,"GCSys":30900224, "LastGC":1631579910873318756,"PauseTotalNs":146651638427, "PauseNs":[], "NumGC":19276} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:39:18 PDT 2021 RSS = 3952880 2021-09-13T17:39:42.757-07:00 [Info] memstats {"Alloc":248393832, "TotalAlloc":3681773235392, "Sys":976283896, "Lookups":0, "Mallocs":239271070,"Frees":238987736, "HeapAlloc":248393832, "HeapSys":934969344, "HeapIdle":682156032, "HeapInuse":252813312,"HeapReleased":433594368, "HeapObjects":283334,"GCSys":30900224, "LastGC":1631579910873318756,"PauseTotalNs":146651638427, "PauseNs":[], "NumGC":19276} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:40:12.784-07:00 [Info] memstats {"Alloc":252009408, "TotalAlloc":3681776850968, "Sys":976283896, "Lookups":0, "Mallocs":239293830,"Frees":238988421, "HeapAlloc":252009408, "HeapSys":934969344, "HeapIdle":678879232, "HeapInuse":256090112,"HeapReleased":433594368, "HeapObjects":305409,"GCSys":30900224, "LastGC":1631579910873318756,"PauseTotalNs":146651638427, "PauseNs":[], "NumGC":19276} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:40:18 PDT 2021 RSS = 3952880 2021-09-13 17:40:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.31200, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 574063809, "lss_used_space": 3249990788, "lss_fragmentation": 82, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 219178909696, "bytes_incoming": 71502511416, "write_amp_avg": 2.00, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 153900612, "recovery_used_space": 706782851, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123521683456, "recovery_lss_frag": 78, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T17:40:42.813-07:00 [Info] memstats {"Alloc":246240416, "TotalAlloc":3681778460192, "Sys":976283896, "Lookups":0, "Mallocs":239312955,"Frees":239055685, "HeapAlloc":246240416, "HeapSys":934969344, "HeapIdle":682229760, "HeapInuse":252739584,"HeapReleased":433594368, "HeapObjects":257270,"GCSys":30900224, "LastGC":1631580030939539789,"PauseTotalNs":146652058348, "PauseNs":[419921], "NumGC":19277} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13 17:40:54 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 17:40:57 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 17:40:59 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13T17:41:12.837-07:00 [Info] memstats {"Alloc":249237704, "TotalAlloc":3681781457480, "Sys":976283896, "Lookups":0, "Mallocs":239332980,"Frees":239055729, "HeapAlloc":249237704, "HeapSys":934969344, "HeapIdle":680280064, "HeapInuse":254689280,"HeapReleased":433594368, "HeapObjects":277251,"GCSys":30900224, "LastGC":1631580030939539789,"PauseTotalNs":146652058348, "PauseNs":[], "NumGC":19277} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:41:19 PDT 2021 RSS = 3952880 2021-09-13 17:41:22 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T17:41:42.864-07:00 [Info] memstats {"Alloc":251238176, "TotalAlloc":3681783457952, "Sys":976283896, "Lookups":0, "Mallocs":239352310,"Frees":239055750, "HeapAlloc":251238176, "HeapSys":934969344, "HeapIdle":679264256, "HeapInuse":255705088,"HeapReleased":433594368, "HeapObjects":296560,"GCSys":30900224, "LastGC":1631580030939539789,"PauseTotalNs":146652058348, "PauseNs":[], "NumGC":19277} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:42:12.894-07:00 [Info] memstats {"Alloc":252775536, "TotalAlloc":3681784995312, "Sys":976283896, "Lookups":0, "Mallocs":239371256,"Frees":239055759, "HeapAlloc":252775536, "HeapSys":934969344, "HeapIdle":678338560, "HeapInuse":256630784,"HeapReleased":433594368, "HeapObjects":315497,"GCSys":30900224, "LastGC":1631580030939539789,"PauseTotalNs":146652058348, "PauseNs":[], "NumGC":19277} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:42:19 PDT 2021 RSS = 3952880 2021-09-13T17:42:42.922-07:00 [Info] memstats {"Alloc":247155848, "TotalAlloc":3681786539080, "Sys":976283896, "Lookups":0, "Mallocs":239390219,"Frees":239121123, "HeapAlloc":247155848, "HeapSys":934969344, "HeapIdle":681361408, "HeapInuse":253607936,"HeapReleased":433594368, "HeapObjects":269096,"GCSys":30900224, "LastGC":1631580151014076466,"PauseTotalNs":146652419110, "PauseNs":[360762], "NumGC":19278} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:43:12.951-07:00 [Info] memstats {"Alloc":248694176, "TotalAlloc":3681788077408, "Sys":976283896, "Lookups":0, "Mallocs":239409164,"Frees":239121132, "HeapAlloc":248694176, "HeapSys":934969344, "HeapIdle":680878080, "HeapInuse":254091264,"HeapReleased":433594368, "HeapObjects":288032,"GCSys":30900224, "LastGC":1631580151014076466,"PauseTotalNs":146652419110, "PauseNs":[], "NumGC":19278} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:43:19 PDT 2021 RSS = 3952880 2021-09-13T17:43:42.977-07:00 [Info] memstats {"Alloc":250238680, "TotalAlloc":3681789621912, "Sys":976283896, "Lookups":0, "Mallocs":239428141,"Frees":239121141, "HeapAlloc":250238680, "HeapSys":934969344, "HeapIdle":680214528, "HeapInuse":254754816,"HeapReleased":433594368, "HeapObjects":307000,"GCSys":30900224, "LastGC":1631580151014076466,"PauseTotalNs":146652419110, "PauseNs":[], "NumGC":19278} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:44:13.004-07:00 [Info] memstats {"Alloc":251781080, "TotalAlloc":3681791164312, "Sys":976283896, "Lookups":0, "Mallocs":239447098,"Frees":239121150, "HeapAlloc":251781080, "HeapSys":934969344, "HeapIdle":679370752, "HeapInuse":255598592,"HeapReleased":433594368, "HeapObjects":325948,"GCSys":30900224, "LastGC":1631580151014076466,"PauseTotalNs":146652419110, "PauseNs":[], "NumGC":19278} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:44:19 PDT 2021 RSS = 3952880 2021-09-13T17:44:43.031-07:00 [Info] memstats {"Alloc":248094312, "TotalAlloc":3681792708096, "Sys":976283896, "Lookups":0, "Mallocs":239466038,"Frees":239185122, "HeapAlloc":248094312, "HeapSys":934969344, "HeapIdle":680452096, "HeapInuse":254517248,"HeapReleased":433594368, "HeapObjects":280916,"GCSys":30900224, "LastGC":1631580271083253664,"PauseTotalNs":146652842636, "PauseNs":[423526], "NumGC":19279} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:45:13.063-07:00 [Info] memstats {"Alloc":249629840, "TotalAlloc":3681794243624, "Sys":976283896, "Lookups":0, "Mallocs":239484987,"Frees":239185131, "HeapAlloc":249629840, "HeapSys":934969344, "HeapIdle":679952384, "HeapInuse":255016960,"HeapReleased":433594368, "HeapObjects":299856,"GCSys":30900224, "LastGC":1631580271083253664,"PauseTotalNs":146652842636, "PauseNs":[], "NumGC":19279} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:45:19 PDT 2021 RSS = 3952880 2021-09-13 17:45:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.32800, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 574063809, "lss_used_space": 3249990788, "lss_fragmentation": 82, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 219178909696, "bytes_incoming": 71502511416, "write_amp_avg": 2.00, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 153900612, "recovery_used_space": 706782851, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123521683456, "recovery_lss_frag": 78, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T17:45:43.095-07:00 [Info] memstats {"Alloc":251281376, "TotalAlloc":3681795895160, "Sys":976283896, "Lookups":0, "Mallocs":239504357,"Frees":239185191, "HeapAlloc":251281376, "HeapSys":934969344, "HeapIdle":679231488, "HeapInuse":255737856,"HeapReleased":433594368, "HeapObjects":319166,"GCSys":30900224, "LastGC":1631580271083253664,"PauseTotalNs":146652842636, "PauseNs":[], "NumGC":19279} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:46:13.122-07:00 [Info] memstats {"Alloc":252820208, "TotalAlloc":3681797433992, "Sys":976283896, "Lookups":0, "Mallocs":239523310,"Frees":239185200, "HeapAlloc":252820208, "HeapSys":934969344, "HeapIdle":678305792, "HeapInuse":256663552,"HeapReleased":433594368, "HeapObjects":338110,"GCSys":30900224, "LastGC":1631580271083253664,"PauseTotalNs":146652842636, "PauseNs":[], "NumGC":19279} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:46:19 PDT 2021 RSS = 3952880 2021-09-13T17:46:43.151-07:00 [Info] memstats {"Alloc":249063144, "TotalAlloc":3681798997152, "Sys":976283896, "Lookups":0, "Mallocs":239542273,"Frees":239249510, "HeapAlloc":249063144, "HeapSys":934969344, "HeapIdle":679460864, "HeapInuse":255508480,"HeapReleased":433594368, "HeapObjects":292763,"GCSys":30900224, "LastGC":1631580391173321918,"PauseTotalNs":146653432693, "PauseNs":[590057], "NumGC":19280} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:47:13.178-07:00 [Info] memstats {"Alloc":250631760, "TotalAlloc":3681800565768, "Sys":976283896, "Lookups":0, "Mallocs":239561218,"Frees":239249519, "HeapAlloc":250631760, "HeapSys":934969344, "HeapIdle":678846464, "HeapInuse":256122880,"HeapReleased":433594368, "HeapObjects":311699,"GCSys":30900224, "LastGC":1631580391173321918,"PauseTotalNs":146653432693, "PauseNs":[], "NumGC":19280} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:47:19 PDT 2021 RSS = 3952880 2021-09-13T17:47:43.211-07:00 [Info] memstats {"Alloc":252193896, "TotalAlloc":3681802127904, "Sys":976283896, "Lookups":0, "Mallocs":239580188,"Frees":239249528, "HeapAlloc":252193896, "HeapSys":934969344, "HeapIdle":678281216, "HeapInuse":256688128,"HeapReleased":433594368, "HeapObjects":330660,"GCSys":30900224, "LastGC":1631580391173321918,"PauseTotalNs":146653432693, "PauseNs":[], "NumGC":19280} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:48:13.237-07:00 [Info] memstats {"Alloc":253743480, "TotalAlloc":3681803677488, "Sys":976283896, "Lookups":0, "Mallocs":239599143,"Frees":239249537, "HeapAlloc":253743480, "HeapSys":934969344, "HeapIdle":677298176, "HeapInuse":257671168,"HeapReleased":433594368, "HeapObjects":349606,"GCSys":30900224, "LastGC":1631580391173321918,"PauseTotalNs":146653432693, "PauseNs":[], "NumGC":19280} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:48:19 PDT 2021 RSS = 3952880 2021-09-13T17:48:43.265-07:00 [Info] memstats {"Alloc":249985912, "TotalAlloc":3681805208408, "Sys":976283896, "Lookups":0, "Mallocs":239618091,"Frees":239313523, "HeapAlloc":249985912, "HeapSys":934969344, "HeapIdle":678510592, "HeapInuse":256458752,"HeapReleased":433594368, "HeapObjects":304568,"GCSys":30900224, "LastGC":1631580511259261809,"PauseTotalNs":146653753925, "PauseNs":[321232], "NumGC":19281} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:49:13.296-07:00 [Info] memstats {"Alloc":251539168, "TotalAlloc":3681806761664, "Sys":976283896, "Lookups":0, "Mallocs":239637049,"Frees":239313532, "HeapAlloc":251539168, "HeapSys":934969344, "HeapIdle":677986304, "HeapInuse":256983040,"HeapReleased":433594368, "HeapObjects":323517,"GCSys":30900224, "LastGC":1631580511259261809,"PauseTotalNs":146653753925, "PauseNs":[], "NumGC":19281} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:49:19 PDT 2021 RSS = 3952880 2021-09-13T17:49:43.325-07:00 [Info] memstats {"Alloc":253061256, "TotalAlloc":3681808283752, "Sys":976283896, "Lookups":0, "Mallocs":239656012,"Frees":239313541, "HeapAlloc":253061256, "HeapSys":934969344, "HeapIdle":677404672, "HeapInuse":257564672,"HeapReleased":433594368, "HeapObjects":342471,"GCSys":30900224, "LastGC":1631580511259261809,"PauseTotalNs":146653753925, "PauseNs":[], "NumGC":19281} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:50:13.352-07:00 [Info] memstats {"Alloc":256725392, "TotalAlloc":3681811947888, "Sys":976283896, "Lookups":0, "Mallocs":239678800,"Frees":239314231, "HeapAlloc":256725392, "HeapSys":934969344, "HeapIdle":674136064, "HeapInuse":260833280,"HeapReleased":433594368, "HeapObjects":364569,"GCSys":30900224, "LastGC":1631580511259261809,"PauseTotalNs":146653753925, "PauseNs":[], "NumGC":19281} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:50:19 PDT 2021 RSS = 3952880 2021-09-13 17:50:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.39400, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 574063809, "lss_used_space": 3250121860, "lss_fragmentation": 82, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 219179040768, "bytes_incoming": 71502511416, "write_amp_avg": 2.00, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 153900612, "recovery_used_space": 706848387, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123521748992, "recovery_lss_frag": 78, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T17:50:43.383-07:00 [Info] memstats {"Alloc":250985632, "TotalAlloc":3681813581816, "Sys":976283896, "Lookups":0, "Mallocs":239697919,"Frees":239381492, "HeapAlloc":250985632, "HeapSys":934969344, "HeapIdle":677535744, "HeapInuse":257433600,"HeapReleased":433594368, "HeapObjects":316427,"GCSys":30900224, "LastGC":1631580631345291026,"PauseTotalNs":146654144796, "PauseNs":[390871], "NumGC":19282} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13 17:50:55 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 17:50:57 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 17:50:59 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13T17:51:13.417-07:00 [Info] memstats {"Alloc":254041368, "TotalAlloc":3681816637552, "Sys":976283896, "Lookups":0, "Mallocs":239717960,"Frees":239381536, "HeapAlloc":254041368, "HeapSys":934969344, "HeapIdle":675454976, "HeapInuse":259514368,"HeapReleased":433594368, "HeapObjects":336424,"GCSys":30900224, "LastGC":1631580631345291026,"PauseTotalNs":146654144796, "PauseNs":[], "NumGC":19282} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:51:19 PDT 2021 RSS = 3952880 2021-09-13 17:51:22 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T17:51:43.445-07:00 [Info] memstats {"Alloc":256080368, "TotalAlloc":3681818676552, "Sys":976283896, "Lookups":0, "Mallocs":239737291,"Frees":239381557, "HeapAlloc":256080368, "HeapSys":934969344, "HeapIdle":674422784, "HeapInuse":260546560,"HeapReleased":433594368, "HeapObjects":355734,"GCSys":30900224, "LastGC":1631580631345291026,"PauseTotalNs":146654144796, "PauseNs":[], "NumGC":19282} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:52:13.484-07:00 [Info] memstats {"Alloc":257658224, "TotalAlloc":3681820254408, "Sys":976283896, "Lookups":0, "Mallocs":239756228,"Frees":239381566, "HeapAlloc":257658224, "HeapSys":934969344, "HeapIdle":673415168, "HeapInuse":261554176,"HeapReleased":433594368, "HeapObjects":374662,"GCSys":30900224, "LastGC":1631580631345291026,"PauseTotalNs":146654144796, "PauseNs":[], "NumGC":19282} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:52:19 PDT 2021 RSS = 3952880 2021-09-13T17:52:43.512-07:00 [Info] memstats {"Alloc":251899944, "TotalAlloc":3681821840368, "Sys":976283896, "Lookups":0, "Mallocs":239775190,"Frees":239446943, "HeapAlloc":251899944, "HeapSys":934969344, "HeapIdle":676618240, "HeapInuse":258351104,"HeapReleased":433594368, "HeapObjects":328247,"GCSys":30900224, "LastGC":1631580751432365767,"PauseTotalNs":146654517970, "PauseNs":[373174], "NumGC":19283} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:53:13.541-07:00 [Info] memstats {"Alloc":253476784, "TotalAlloc":3681823417208, "Sys":976283896, "Lookups":0, "Mallocs":239794142,"Frees":239446952, "HeapAlloc":253476784, "HeapSys":934969344, "HeapIdle":676020224, "HeapInuse":258949120,"HeapReleased":433594368, "HeapObjects":347190,"GCSys":30900224, "LastGC":1631580751432365767,"PauseTotalNs":146654517970, "PauseNs":[], "NumGC":19283} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:53:19 PDT 2021 RSS = 3952880 2021-09-13T17:53:43.567-07:00 [Info] memstats {"Alloc":255035000, "TotalAlloc":3681824975424, "Sys":976283896, "Lookups":0, "Mallocs":239813112,"Frees":239446961, "HeapAlloc":255035000, "HeapSys":934969344, "HeapIdle":675430400, "HeapInuse":259538944,"HeapReleased":433594368, "HeapObjects":366151,"GCSys":30900224, "LastGC":1631580751432365767,"PauseTotalNs":146654517970, "PauseNs":[], "NumGC":19283} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:54:13.604-07:00 [Info] memstats {"Alloc":256590392, "TotalAlloc":3681826530816, "Sys":976283896, "Lookups":0, "Mallocs":239832069,"Frees":239446970, "HeapAlloc":256590392, "HeapSys":934969344, "HeapIdle":674496512, "HeapInuse":260472832,"HeapReleased":433594368, "HeapObjects":385099,"GCSys":30900224, "LastGC":1631580751432365767,"PauseTotalNs":146654517970, "PauseNs":[], "NumGC":19283} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:54:19 PDT 2021 RSS = 3952880 2021-09-13T17:54:43.631-07:00 [Info] memstats {"Alloc":252834792, "TotalAlloc":3681828099704, "Sys":976283896, "Lookups":0, "Mallocs":239851038,"Frees":239510959, "HeapAlloc":252834792, "HeapSys":934969344, "HeapIdle":675700736, "HeapInuse":259268608,"HeapReleased":433594368, "HeapObjects":340079,"GCSys":30900224, "LastGC":1631580871513202977,"PauseTotalNs":146654871429, "PauseNs":[353459], "NumGC":19284} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:55:13.658-07:00 [Info] memstats {"Alloc":254397664, "TotalAlloc":3681829662576, "Sys":976283896, "Lookups":0, "Mallocs":239869992,"Frees":239510968, "HeapAlloc":254397664, "HeapSys":934969344, "HeapIdle":675045376, "HeapInuse":259923968,"HeapReleased":433594368, "HeapObjects":359024,"GCSys":30900224, "LastGC":1631580871513202977,"PauseTotalNs":146654871429, "PauseNs":[], "NumGC":19284} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:55:19 PDT 2021 RSS = 3952880 2021-09-13 17:55:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.29600, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 574063809, "lss_used_space": 3250121860, "lss_fragmentation": 82, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 219179040768, "bytes_incoming": 71502511416, "write_amp_avg": 2.00, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 153900612, "recovery_used_space": 706848387, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123521748992, "recovery_lss_frag": 78, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T17:55:43.685-07:00 [Info] memstats {"Alloc":255998296, "TotalAlloc":3681831263208, "Sys":976283896, "Lookups":0, "Mallocs":239889114,"Frees":239511013, "HeapAlloc":255998296, "HeapSys":934969344, "HeapIdle":674471936, "HeapInuse":260497408,"HeapReleased":433594368, "HeapObjects":378101,"GCSys":30900224, "LastGC":1631580871513202977,"PauseTotalNs":146654871429, "PauseNs":[], "NumGC":19284} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:56:13.712-07:00 [Info] memstats {"Alloc":257537032, "TotalAlloc":3681832801944, "Sys":976283896, "Lookups":0, "Mallocs":239908061,"Frees":239511022, "HeapAlloc":257537032, "HeapSys":934969344, "HeapIdle":673423360, "HeapInuse":261545984,"HeapReleased":433594368, "HeapObjects":397039,"GCSys":30900224, "LastGC":1631580871513202977,"PauseTotalNs":146654871429, "PauseNs":[], "NumGC":19284} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:56:19 PDT 2021 RSS = 3952880 2021-09-13T17:56:43.743-07:00 [Info] memstats {"Alloc":253768904, "TotalAlloc":3681834331344, "Sys":976283896, "Lookups":0, "Mallocs":239927033,"Frees":239575140, "HeapAlloc":253768904, "HeapSys":934969344, "HeapIdle":674701312, "HeapInuse":260268032,"HeapReleased":433594368, "HeapObjects":351893,"GCSys":30900224, "LastGC":1631580991604942687,"PauseTotalNs":146655194685, "PauseNs":[323256], "NumGC":19285} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:57:13.770-07:00 [Info] memstats {"Alloc":255299552, "TotalAlloc":3681835861992, "Sys":976283896, "Lookups":0, "Mallocs":239945970,"Frees":239575149, "HeapAlloc":255299552, "HeapSys":934969344, "HeapIdle":674201600, "HeapInuse":260767744,"HeapReleased":433594368, "HeapObjects":370821,"GCSys":30900224, "LastGC":1631580991604942687,"PauseTotalNs":146655194685, "PauseNs":[], "NumGC":19285} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:57:19 PDT 2021 RSS = 3952880 2021-09-13T17:57:43.797-07:00 [Info] memstats {"Alloc":256849272, "TotalAlloc":3681837411712, "Sys":976283896, "Lookups":0, "Mallocs":239964934,"Frees":239575158, "HeapAlloc":256849272, "HeapSys":934969344, "HeapIdle":673570816, "HeapInuse":261398528,"HeapReleased":433594368, "HeapObjects":389776,"GCSys":30900224, "LastGC":1631580991604942687,"PauseTotalNs":146655194685, "PauseNs":[], "NumGC":19285} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:58:13.825-07:00 [Info] memstats {"Alloc":258411224, "TotalAlloc":3681838973664, "Sys":976283896, "Lookups":0, "Mallocs":239983880,"Frees":239575167, "HeapAlloc":258411224, "HeapSys":934969344, "HeapIdle":672661504, "HeapInuse":262307840,"HeapReleased":433594368, "HeapObjects":408713,"GCSys":30900224, "LastGC":1631580991604942687,"PauseTotalNs":146655194685, "PauseNs":[], "NumGC":19285} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:58:20 PDT 2021 RSS = 3952880 2021-09-13T17:58:43.849-07:00 [Info] memstats {"Alloc":254724392, "TotalAlloc":3681840537528, "Sys":976283896, "Lookups":0, "Mallocs":240002839,"Frees":239639125, "HeapAlloc":254724392, "HeapSys":934969344, "HeapIdle":673783808, "HeapInuse":261185536,"HeapReleased":433594368, "HeapObjects":363714,"GCSys":30900224, "LastGC":1631581111700407684,"PauseTotalNs":146655589980, "PauseNs":[395295], "NumGC":19286} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T17:59:13.876-07:00 [Info] memstats {"Alloc":256306512, "TotalAlloc":3681842119648, "Sys":976283896, "Lookups":0, "Mallocs":240021794,"Frees":239639134, "HeapAlloc":256306512, "HeapSys":934969344, "HeapIdle":673169408, "HeapInuse":261799936,"HeapReleased":433594368, "HeapObjects":382660,"GCSys":30900224, "LastGC":1631581111700407684,"PauseTotalNs":146655589980, "PauseNs":[], "NumGC":19286} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 17:59:20 PDT 2021 RSS = 3952880 2021-09-13T17:59:43.903-07:00 [Info] memstats {"Alloc":257851912, "TotalAlloc":3681843665048, "Sys":976283896, "Lookups":0, "Mallocs":240040759,"Frees":239639143, "HeapAlloc":257851912, "HeapSys":934969344, "HeapIdle":672669696, "HeapInuse":262299648,"HeapReleased":433594368, "HeapObjects":401616,"GCSys":30900224, "LastGC":1631581111700407684,"PauseTotalNs":146655589980, "PauseNs":[], "NumGC":19286} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:00:13.930-07:00 [Info] memstats {"Alloc":261475968, "TotalAlloc":3681847289104, "Sys":976283896, "Lookups":0, "Mallocs":240063569,"Frees":239639836, "HeapAlloc":261475968, "HeapSys":934969344, "HeapIdle":669417472, "HeapInuse":265551872,"HeapReleased":433594368, "HeapObjects":423733,"GCSys":30900224, "LastGC":1631581111700407684,"PauseTotalNs":146655589980, "PauseNs":[], "NumGC":19286} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:00:20 PDT 2021 RSS = 3952880 2021-09-13 18:00:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.26200, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 574063809, "lss_used_space": 3250121860, "lss_fragmentation": 82, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 219179040768, "bytes_incoming": 71502511416, "write_amp_avg": 2.00, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 153900612, "recovery_used_space": 706848387, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123521748992, "recovery_lss_frag": 78, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T18:00:43.958-07:00 [Info] memstats {"Alloc":255686384, "TotalAlloc":3681848952400, "Sys":976283896, "Lookups":0, "Mallocs":240082922,"Frees":239707374, "HeapAlloc":255686384, "HeapSys":934969344, "HeapIdle":672751616, "HeapInuse":262217728,"HeapReleased":433594368, "HeapObjects":375548,"GCSys":30900224, "LastGC":1631581231817153790,"PauseTotalNs":146655975581, "PauseNs":[385601], "NumGC":19287} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13 18:00:55 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 18:00:57 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 18:00:59 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13T18:01:13.984-07:00 [Info] memstats {"Alloc":258705288, "TotalAlloc":3681851971304, "Sys":976283896, "Lookups":0, "Mallocs":240102955,"Frees":239707417, "HeapAlloc":258705288, "HeapSys":934969344, "HeapIdle":670834688, "HeapInuse":264134656,"HeapReleased":433594368, "HeapObjects":395538,"GCSys":30900224, "LastGC":1631581231817153790,"PauseTotalNs":146655975581, "PauseNs":[], "NumGC":19287} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:01:20 PDT 2021 RSS = 3952880 2021-09-13 18:01:23 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T18:01:44.013-07:00 [Info] memstats {"Alloc":260728480, "TotalAlloc":3681853994496, "Sys":976283896, "Lookups":0, "Mallocs":240122275,"Frees":239707438, "HeapAlloc":260728480, "HeapSys":934969344, "HeapIdle":669745152, "HeapInuse":265224192,"HeapReleased":433594368, "HeapObjects":414837,"GCSys":30900224, "LastGC":1631581231817153790,"PauseTotalNs":146655975581, "PauseNs":[], "NumGC":19287} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:02:14.040-07:00 [Info] memstats {"Alloc":262288688, "TotalAlloc":3681855554704, "Sys":976283896, "Lookups":0, "Mallocs":240141225,"Frees":239707447, "HeapAlloc":262288688, "HeapSys":934969344, "HeapIdle":668770304, "HeapInuse":266199040,"HeapReleased":433594368, "HeapObjects":433778,"GCSys":30900224, "LastGC":1631581231817153790,"PauseTotalNs":146655975581, "PauseNs":[], "NumGC":19287} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:02:20 PDT 2021 RSS = 3952880 2021-09-13T18:02:44.069-07:00 [Info] memstats {"Alloc":256608024, "TotalAlloc":3681857105896, "Sys":976283896, "Lookups":0, "Mallocs":240160198,"Frees":239772850, "HeapAlloc":256608024, "HeapSys":934969344, "HeapIdle":671907840, "HeapInuse":263061504,"HeapReleased":433594368, "HeapObjects":387348,"GCSys":30900224, "LastGC":1631581351915958273,"PauseTotalNs":146656344160, "PauseNs":[368579], "NumGC":19288} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:03:14.100-07:00 [Info] memstats {"Alloc":258189936, "TotalAlloc":3681858687808, "Sys":976283896, "Lookups":0, "Mallocs":240179159,"Frees":239772859, "HeapAlloc":258189936, "HeapSys":934969344, "HeapIdle":671350784, "HeapInuse":263618560,"HeapReleased":433594368, "HeapObjects":406300,"GCSys":30900224, "LastGC":1631581351915958273,"PauseTotalNs":146656344160, "PauseNs":[], "NumGC":19288} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:03:20 PDT 2021 RSS = 3952880 2021-09-13T18:03:44.127-07:00 [Info] memstats {"Alloc":259790344, "TotalAlloc":3681860288216, "Sys":976283896, "Lookups":0, "Mallocs":240198115,"Frees":239772868, "HeapAlloc":259790344, "HeapSys":934969344, "HeapIdle":670654464, "HeapInuse":264314880,"HeapReleased":433594368, "HeapObjects":425247,"GCSys":30900224, "LastGC":1631581351915958273,"PauseTotalNs":146656344160, "PauseNs":[], "NumGC":19288} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:04:14.155-07:00 [Info] memstats {"Alloc":261408664, "TotalAlloc":3681861906536, "Sys":976283896, "Lookups":0, "Mallocs":240217074,"Frees":239772877, "HeapAlloc":261408664, "HeapSys":934969344, "HeapIdle":669573120, "HeapInuse":265396224,"HeapReleased":433594368, "HeapObjects":444197,"GCSys":30900224, "LastGC":1631581351915958273,"PauseTotalNs":146656344160, "PauseNs":[], "NumGC":19288} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:04:20 PDT 2021 RSS = 3952880 2021-09-13T18:04:44.183-07:00 [Info] memstats {"Alloc":257579528, "TotalAlloc":3681863512704, "Sys":976283896, "Lookups":0, "Mallocs":240236056,"Frees":239836864, "HeapAlloc":257579528, "HeapSys":934969344, "HeapIdle":670949376, "HeapInuse":264019968,"HeapReleased":433594368, "HeapObjects":399192,"GCSys":30900224, "LastGC":1631581471999881791,"PauseTotalNs":146656985469, "PauseNs":[641309], "NumGC":19289} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:05:14.207-07:00 [Info] memstats {"Alloc":259186624, "TotalAlloc":3681865119800, "Sys":976283896, "Lookups":0, "Mallocs":240255009,"Frees":239836873, "HeapAlloc":259186624, "HeapSys":934969344, "HeapIdle":670253056, "HeapInuse":264716288,"HeapReleased":433594368, "HeapObjects":418136,"GCSys":30900224, "LastGC":1631581471999881791,"PauseTotalNs":146656985469, "PauseNs":[], "NumGC":19289} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:05:20 PDT 2021 RSS = 3952880 2021-09-13 18:05:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.30600, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 574063809, "lss_used_space": 3250187396, "lss_fragmentation": 82, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 219179106304, "bytes_incoming": 71502511416, "write_amp_avg": 2.00, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 153900612, "recovery_used_space": 706881155, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123521781760, "recovery_lss_frag": 78, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T18:05:44.233-07:00 [Info] memstats {"Alloc":260872904, "TotalAlloc":3681866806080, "Sys":976283896, "Lookups":0, "Mallocs":240274154,"Frees":239836918, "HeapAlloc":260872904, "HeapSys":934969344, "HeapIdle":669442048, "HeapInuse":265527296,"HeapReleased":433594368, "HeapObjects":437236,"GCSys":30900224, "LastGC":1631581471999881791,"PauseTotalNs":146656985469, "PauseNs":[], "NumGC":19289} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:06:14.265-07:00 [Info] memstats {"Alloc":262509736, "TotalAlloc":3681868442912, "Sys":976283896, "Lookups":0, "Mallocs":240293107,"Frees":239836927, "HeapAlloc":262509736, "HeapSys":934969344, "HeapIdle":668409856, "HeapInuse":266559488,"HeapReleased":433594368, "HeapObjects":456180,"GCSys":30900224, "LastGC":1631581471999881791,"PauseTotalNs":146656985469, "PauseNs":[], "NumGC":19289} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:06:20 PDT 2021 RSS = 3952880 2021-09-13T18:06:44.293-07:00 [Info] memstats {"Alloc":258532232, "TotalAlloc":3681870057544, "Sys":976283896, "Lookups":0, "Mallocs":240312087,"Frees":239901071, "HeapAlloc":258532232, "HeapSys":934969344, "HeapIdle":669966336, "HeapInuse":265003008,"HeapReleased":433594368, "HeapObjects":411016,"GCSys":30900224, "LastGC":1631581592104462157,"PauseTotalNs":146657407283, "PauseNs":[421814], "NumGC":19290} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:07:14.322-07:00 [Info] memstats {"Alloc":260145856, "TotalAlloc":3681871671168, "Sys":976283896, "Lookups":0, "Mallocs":240331042,"Frees":239901080, "HeapAlloc":260145856, "HeapSys":934969344, "HeapIdle":669368320, "HeapInuse":265601024,"HeapReleased":433594368, "HeapObjects":429962,"GCSys":30900224, "LastGC":1631581592104462157,"PauseTotalNs":146657407283, "PauseNs":[], "NumGC":19290} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:07:20 PDT 2021 RSS = 3952880 2021-09-13T18:07:44.348-07:00 [Info] memstats {"Alloc":261754952, "TotalAlloc":3681873280264, "Sys":976283896, "Lookups":0, "Mallocs":240350014,"Frees":239901089, "HeapAlloc":261754952, "HeapSys":934969344, "HeapIdle":668663808, "HeapInuse":266305536,"HeapReleased":433594368, "HeapObjects":448925,"GCSys":30900224, "LastGC":1631581592104462157,"PauseTotalNs":146657407283, "PauseNs":[], "NumGC":19290} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:08:14.373-07:00 [Info] memstats {"Alloc":263365864, "TotalAlloc":3681874891176, "Sys":976283896, "Lookups":0, "Mallocs":240368960,"Frees":239901098, "HeapAlloc":263365864, "HeapSys":934969344, "HeapIdle":667656192, "HeapInuse":267313152,"HeapReleased":433594368, "HeapObjects":467862,"GCSys":30900224, "LastGC":1631581592104462157,"PauseTotalNs":146657407283, "PauseNs":[], "NumGC":19290} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:08:20 PDT 2021 RSS = 3952880 2021-09-13T18:08:44.400-07:00 [Info] memstats {"Alloc":259471144, "TotalAlloc":3681876495520, "Sys":976283896, "Lookups":0, "Mallocs":240387930,"Frees":239965099, "HeapAlloc":259471144, "HeapSys":934969344, "HeapIdle":668991488, "HeapInuse":265977856,"HeapReleased":433594368, "HeapObjects":422831,"GCSys":30900224, "LastGC":1631581712199820389,"PauseTotalNs":146657721460, "PauseNs":[314177], "NumGC":19291} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:09:14.427-07:00 [Info] memstats {"Alloc":261083360, "TotalAlloc":3681878107736, "Sys":976283896, "Lookups":0, "Mallocs":240406889,"Frees":239965108, "HeapAlloc":261083360, "HeapSys":934969344, "HeapIdle":668475392, "HeapInuse":266493952,"HeapReleased":433594368, "HeapObjects":441781,"GCSys":30900224, "LastGC":1631581712199820389,"PauseTotalNs":146657721460, "PauseNs":[], "NumGC":19291} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:09:20 PDT 2021 RSS = 3952880 2021-09-13T18:09:44.460-07:00 [Info] memstats {"Alloc":262662872, "TotalAlloc":3681879687248, "Sys":976283896, "Lookups":0, "Mallocs":240425861,"Frees":239965117, "HeapAlloc":262662872, "HeapSys":934969344, "HeapIdle":667836416, "HeapInuse":267132928,"HeapReleased":433594368, "HeapObjects":460744,"GCSys":30900224, "LastGC":1631581712199820389,"PauseTotalNs":146657721460, "PauseNs":[], "NumGC":19291} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:10:14.487-07:00 [Info] memstats {"Alloc":266344936, "TotalAlloc":3681883369312, "Sys":976283896, "Lookups":0, "Mallocs":240448661,"Frees":239965810, "HeapAlloc":266344936, "HeapSys":934969344, "HeapIdle":664510464, "HeapInuse":270458880,"HeapReleased":433594368, "HeapObjects":482851,"GCSys":30900224, "LastGC":1631581712199820389,"PauseTotalNs":146657721460, "PauseNs":[], "NumGC":19291} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:10:20 PDT 2021 RSS = 3952880 2021-09-13 18:10:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.28400, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 574063809, "lss_used_space": 3250187396, "lss_fragmentation": 82, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 219179106304, "bytes_incoming": 71502511416, "write_amp_avg": 2.00, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 153900612, "recovery_used_space": 706881155, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123521781760, "recovery_lss_frag": 78, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T18:10:44.519-07:00 [Info] memstats {"Alloc":260460304, "TotalAlloc":3681885054488, "Sys":976283896, "Lookups":0, "Mallocs":240467791,"Frees":240033114, "HeapAlloc":260460304, "HeapSys":934969344, "HeapIdle":667901952, "HeapInuse":267067392,"HeapReleased":433594368, "HeapObjects":434677,"GCSys":30900224, "LastGC":1631581832307698807,"PauseTotalNs":146658077588, "PauseNs":[356128], "NumGC":19292} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13 18:10:55 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 18:10:57 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 18:10:59 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13T18:11:14.551-07:00 [Info] memstats {"Alloc":263557112, "TotalAlloc":3681888151296, "Sys":976283896, "Lookups":0, "Mallocs":240487818,"Frees":240033158, "HeapAlloc":263557112, "HeapSys":934969344, "HeapIdle":665812992, "HeapInuse":269156352,"HeapReleased":433594368, "HeapObjects":454660,"GCSys":30900224, "LastGC":1631581832307698807,"PauseTotalNs":146658077588, "PauseNs":[], "NumGC":19292} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:11:20 PDT 2021 RSS = 3952880 2021-09-13 18:11:23 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T18:11:44.579-07:00 [Info] memstats {"Alloc":265613984, "TotalAlloc":3681890208168, "Sys":976283896, "Lookups":0, "Mallocs":240507141,"Frees":240033179, "HeapAlloc":265613984, "HeapSys":934969344, "HeapIdle":664772608, "HeapInuse":270196736,"HeapReleased":433594368, "HeapObjects":473962,"GCSys":30900224, "LastGC":1631581832307698807,"PauseTotalNs":146658077588, "PauseNs":[], "NumGC":19292} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:12:14.605-07:00 [Info] memstats {"Alloc":267202096, "TotalAlloc":3681891796280, "Sys":976283896, "Lookups":0, "Mallocs":240526088,"Frees":240033188, "HeapAlloc":267202096, "HeapSys":934969344, "HeapIdle":663896064, "HeapInuse":271073280,"HeapReleased":433594368, "HeapObjects":492900,"GCSys":30900224, "LastGC":1631581832307698807,"PauseTotalNs":146658077588, "PauseNs":[], "NumGC":19292} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:12:21 PDT 2021 RSS = 3952880 2021-09-13T18:12:44.633-07:00 [Info] memstats {"Alloc":261376648, "TotalAlloc":3681893416800, "Sys":976283896, "Lookups":0, "Mallocs":240545057,"Frees":240098570, "HeapAlloc":261376648, "HeapSys":934969344, "HeapIdle":667074560, "HeapInuse":267894784,"HeapReleased":433594368, "HeapObjects":446487,"GCSys":30900224, "LastGC":1631581952413131342,"PauseTotalNs":146658506725, "PauseNs":[429137], "NumGC":19293} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:13:14.662-07:00 [Info] memstats {"Alloc":263003152, "TotalAlloc":3681895043304, "Sys":976283896, "Lookups":0, "Mallocs":240564020,"Frees":240098579, "HeapAlloc":263003152, "HeapSys":934969344, "HeapIdle":666394624, "HeapInuse":268574720,"HeapReleased":433594368, "HeapObjects":465441,"GCSys":30900224, "LastGC":1631581952413131342,"PauseTotalNs":146658506725, "PauseNs":[], "NumGC":19293} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:13:21 PDT 2021 RSS = 3952880 2021-09-13T18:13:44.689-07:00 [Info] memstats {"Alloc":264626056, "TotalAlloc":3681896666208, "Sys":976283896, "Lookups":0, "Mallocs":240582985,"Frees":240098588, "HeapAlloc":264626056, "HeapSys":934969344, "HeapIdle":665796608, "HeapInuse":269172736,"HeapReleased":433594368, "HeapObjects":484397,"GCSys":30900224, "LastGC":1631581952413131342,"PauseTotalNs":146658506725, "PauseNs":[], "NumGC":19293} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:14:14.721-07:00 [Info] memstats {"Alloc":266266872, "TotalAlloc":3681898307024, "Sys":976283896, "Lookups":0, "Mallocs":240601934,"Frees":240098597, "HeapAlloc":266266872, "HeapSys":934969344, "HeapIdle":664682496, "HeapInuse":270286848,"HeapReleased":433594368, "HeapObjects":503337,"GCSys":30900224, "LastGC":1631581952413131342,"PauseTotalNs":146658506725, "PauseNs":[], "NumGC":19293} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:14:21 PDT 2021 RSS = 3952880 2021-09-13T18:14:44.747-07:00 [Info] memstats {"Alloc":262323112, "TotalAlloc":3681899951672, "Sys":976283896, "Lookups":0, "Mallocs":240620910,"Frees":240162604, "HeapAlloc":262323112, "HeapSys":934969344, "HeapIdle":666173440, "HeapInuse":268795904,"HeapReleased":433594368, "HeapObjects":458306,"GCSys":30900224, "LastGC":1631582072528233730,"PauseTotalNs":146658864434, "PauseNs":[357709], "NumGC":19294} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:15:14.774-07:00 [Info] memstats {"Alloc":263956032, "TotalAlloc":3681901584592, "Sys":976283896, "Lookups":0, "Mallocs":240639861,"Frees":240162613, "HeapAlloc":263956032, "HeapSys":934969344, "HeapIdle":665550848, "HeapInuse":269418496,"HeapReleased":433594368, "HeapObjects":477248,"GCSys":30900224, "LastGC":1631582072528233730,"PauseTotalNs":146658864434, "PauseNs":[], "NumGC":19294} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:15:21 PDT 2021 RSS = 3952880 2021-09-13 18:15:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.26965, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 574063809, "lss_used_space": 3250187396, "lss_fragmentation": 82, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 219179106304, "bytes_incoming": 71502511416, "write_amp_avg": 2.00, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 153900612, "recovery_used_space": 706881155, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123521781760, "recovery_lss_frag": 78, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T18:15:44.803-07:00 [Info] memstats {"Alloc":265700816, "TotalAlloc":3681903329376, "Sys":976283896, "Lookups":0, "Mallocs":240659227,"Frees":240162673, "HeapAlloc":265700816, "HeapSys":934969344, "HeapIdle":664788992, "HeapInuse":270180352,"HeapReleased":433594368, "HeapObjects":496554,"GCSys":30900224, "LastGC":1631582072528233730,"PauseTotalNs":146658864434, "PauseNs":[], "NumGC":19294} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:16:14.830-07:00 [Info] memstats {"Alloc":267344800, "TotalAlloc":3681904973360, "Sys":976283896, "Lookups":0, "Mallocs":240678184,"Frees":240162682, "HeapAlloc":267344800, "HeapSys":934969344, "HeapIdle":663633920, "HeapInuse":271335424,"HeapReleased":433594368, "HeapObjects":515502,"GCSys":30900224, "LastGC":1631582072528233730,"PauseTotalNs":146658864434, "PauseNs":[], "NumGC":19294} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:16:21 PDT 2021 RSS = 3952880 2021-09-13T18:16:44.857-07:00 [Info] memstats {"Alloc":263263704, "TotalAlloc":3681906597256, "Sys":976283896, "Lookups":0, "Mallocs":240697152,"Frees":240227028, "HeapAlloc":263263704, "HeapSys":934969344, "HeapIdle":665174016, "HeapInuse":269795328,"HeapReleased":433594368, "HeapObjects":470124,"GCSys":30900224, "LastGC":1631582192654266390,"PauseTotalNs":146659189847, "PauseNs":[325413], "NumGC":19295} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:17:14.884-07:00 [Info] memstats {"Alloc":264876784, "TotalAlloc":3681908210336, "Sys":976283896, "Lookups":0, "Mallocs":240716111,"Frees":240227037, "HeapAlloc":264876784, "HeapSys":934969344, "HeapIdle":664567808, "HeapInuse":270401536,"HeapReleased":433594368, "HeapObjects":489074,"GCSys":30900224, "LastGC":1631582192654266390,"PauseTotalNs":146659189847, "PauseNs":[], "NumGC":19295} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:17:21 PDT 2021 RSS = 3952880 2021-09-13T18:17:44.926-07:00 [Info] memstats {"Alloc":266533352, "TotalAlloc":3681909866904, "Sys":976283896, "Lookups":0, "Mallocs":240735079,"Frees":240227046, "HeapAlloc":266533352, "HeapSys":934969344, "HeapIdle":663846912, "HeapInuse":271122432,"HeapReleased":433594368, "HeapObjects":508033,"GCSys":30900224, "LastGC":1631582192654266390,"PauseTotalNs":146659189847, "PauseNs":[], "NumGC":19295} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:18:14.953-07:00 [Info] memstats {"Alloc":268211288, "TotalAlloc":3681911544840, "Sys":976283896, "Lookups":0, "Mallocs":240754028,"Frees":240227055, "HeapAlloc":268211288, "HeapSys":934969344, "HeapIdle":662847488, "HeapInuse":272121856,"HeapReleased":433594368, "HeapObjects":526973,"GCSys":30900224, "LastGC":1631582192654266390,"PauseTotalNs":146659189847, "PauseNs":[], "NumGC":19295} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:18:21 PDT 2021 RSS = 3952880 2021-09-13T18:18:44.982-07:00 [Info] memstats {"Alloc":264228664, "TotalAlloc":3681913222160, "Sys":976283896, "Lookups":0, "Mallocs":240773010,"Frees":240291061, "HeapAlloc":264228664, "HeapSys":934969344, "HeapIdle":664248320, "HeapInuse":270721024,"HeapReleased":433594368, "HeapObjects":481949,"GCSys":30900224, "LastGC":1631582312792141167,"PauseTotalNs":146659469107, "PauseNs":[279260], "NumGC":19296} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:19:15.014-07:00 [Info] memstats {"Alloc":265899136, "TotalAlloc":3681914892632, "Sys":976283896, "Lookups":0, "Mallocs":240791963,"Frees":240291070, "HeapAlloc":265899136, "HeapSys":934969344, "HeapIdle":663527424, "HeapInuse":271441920,"HeapReleased":433594368, "HeapObjects":500893,"GCSys":30900224, "LastGC":1631582312792141167,"PauseTotalNs":146659469107, "PauseNs":[], "NumGC":19296} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:19:21 PDT 2021 RSS = 3952880 2021-09-13T18:19:45.042-07:00 [Info] memstats {"Alloc":267577912, "TotalAlloc":3681916571408, "Sys":976283896, "Lookups":0, "Mallocs":240810940,"Frees":240291079, "HeapAlloc":267577912, "HeapSys":934969344, "HeapIdle":662880256, "HeapInuse":272089088,"HeapReleased":433594368, "HeapObjects":519861,"GCSys":30900224, "LastGC":1631582312792141167,"PauseTotalNs":146659469107, "PauseNs":[], "NumGC":19296} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:20:15.075-07:00 [Info] memstats {"Alloc":271355168, "TotalAlloc":3681920348664, "Sys":976283896, "Lookups":0, "Mallocs":240833754,"Frees":240291774, "HeapAlloc":271355168, "HeapSys":934969344, "HeapIdle":659357696, "HeapInuse":275611648,"HeapReleased":433594368, "HeapObjects":541980,"GCSys":30900224, "LastGC":1631582312792141167,"PauseTotalNs":146659469107, "PauseNs":[], "NumGC":19296} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:20:21 PDT 2021 RSS = 3953144 2021-09-13 18:20:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.37200, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 574063809, "lss_used_space": 3250318468, "lss_fragmentation": 82, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 219179237376, "bytes_incoming": 71502511416, "write_amp_avg": 2.00, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 153900612, "recovery_used_space": 706946691, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123521847296, "recovery_lss_frag": 78, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T18:20:45.106-07:00 [Info] memstats {"Alloc":265195952, "TotalAlloc":3681922077248, "Sys":976283896, "Lookups":0, "Mallocs":240852889,"Frees":240359125, "HeapAlloc":265195952, "HeapSys":934969344, "HeapIdle":663085056, "HeapInuse":271884288,"HeapReleased":433594368, "HeapObjects":493764,"GCSys":30900224, "LastGC":1631582432915415835,"PauseTotalNs":146659845176, "PauseNs":[376069], "NumGC":19297} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13 18:20:55 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 18:20:57 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 18:20:59 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13T18:21:15.138-07:00 [Info] memstats {"Alloc":268344776, "TotalAlloc":3681925226072, "Sys":976283896, "Lookups":0, "Mallocs":240872926,"Frees":240359169, "HeapAlloc":268344776, "HeapSys":934969344, "HeapIdle":661012480, "HeapInuse":273956864,"HeapReleased":433594368, "HeapObjects":513757,"GCSys":30900224, "LastGC":1631582432915415835,"PauseTotalNs":146659845176, "PauseNs":[], "NumGC":19297} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:21:21 PDT 2021 RSS = 3953144 2021-09-13 18:21:23 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T18:21:45.162-07:00 [Info] memstats {"Alloc":270496208, "TotalAlloc":3681927377504, "Sys":976283896, "Lookups":0, "Mallocs":240892251,"Frees":240359190, "HeapAlloc":270496208, "HeapSys":934969344, "HeapIdle":659865600, "HeapInuse":275103744,"HeapReleased":433594368, "HeapObjects":533061,"GCSys":30900224, "LastGC":1631582432915415835,"PauseTotalNs":146659845176, "PauseNs":[], "NumGC":19297} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:22:15.190-07:00 [Info] memstats {"Alloc":272171376, "TotalAlloc":3681929052672, "Sys":976283896, "Lookups":0, "Mallocs":240911201,"Frees":240359199, "HeapAlloc":272171376, "HeapSys":934969344, "HeapIdle":658800640, "HeapInuse":276168704,"HeapReleased":433594368, "HeapObjects":552002,"GCSys":30900224, "LastGC":1631582432915415835,"PauseTotalNs":146659845176, "PauseNs":[], "NumGC":19297} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:22:21 PDT 2021 RSS = 3953144 2021-09-13T18:22:45.218-07:00 [Info] memstats {"Alloc":266120792, "TotalAlloc":3681930724728, "Sys":976283896, "Lookups":0, "Mallocs":240930183,"Frees":240424607, "HeapAlloc":266120792, "HeapSys":934969344, "HeapIdle":662323200, "HeapInuse":272646144,"HeapReleased":433594368, "HeapObjects":505576,"GCSys":30900224, "LastGC":1631582553030493035,"PauseTotalNs":146660241216, "PauseNs":[396040], "NumGC":19298} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:23:15.250-07:00 [Info] memstats {"Alloc":267804896, "TotalAlloc":3681932408832, "Sys":976283896, "Lookups":0, "Mallocs":240949140,"Frees":240424616, "HeapAlloc":267804896, "HeapSys":934969344, "HeapIdle":661602304, "HeapInuse":273367040,"HeapReleased":433594368, "HeapObjects":524524,"GCSys":30900224, "LastGC":1631582553030493035,"PauseTotalNs":146660241216, "PauseNs":[], "NumGC":19298} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:23:21 PDT 2021 RSS = 3953144 2021-09-13T18:23:45.280-07:00 [Info] memstats {"Alloc":269508312, "TotalAlloc":3681934112248, "Sys":976283896, "Lookups":0, "Mallocs":240968120,"Frees":240424625, "HeapAlloc":269508312, "HeapSys":934969344, "HeapIdle":660807680, "HeapInuse":274161664,"HeapReleased":433594368, "HeapObjects":543495,"GCSys":30900224, "LastGC":1631582553030493035,"PauseTotalNs":146660241216, "PauseNs":[], "NumGC":19298} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:24:15.308-07:00 [Info] memstats {"Alloc":271212760, "TotalAlloc":3681935816696, "Sys":976283896, "Lookups":0, "Mallocs":240987081,"Frees":240424634, "HeapAlloc":271212760, "HeapSys":934969344, "HeapIdle":659726336, "HeapInuse":275243008,"HeapReleased":433594368, "HeapObjects":562447,"GCSys":30900224, "LastGC":1631582553030493035,"PauseTotalNs":146660241216, "PauseNs":[], "NumGC":19298} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:24:21 PDT 2021 RSS = 3953144 2021-09-13T18:24:45.338-07:00 [Info] memstats {"Alloc":267078152, "TotalAlloc":3681937516480, "Sys":976283896, "Lookups":0, "Mallocs":241006056,"Frees":240488669, "HeapAlloc":267078152, "HeapSys":934969344, "HeapIdle":661274624, "HeapInuse":273694720,"HeapReleased":433594368, "HeapObjects":517387,"GCSys":30900224, "LastGC":1631582673155988058,"PauseTotalNs":146660601559, "PauseNs":[360343], "NumGC":19299} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:25:15.368-07:00 [Info] memstats {"Alloc":268768720, "TotalAlloc":3681939207048, "Sys":976283896, "Lookups":0, "Mallocs":241025013,"Frees":240488678, "HeapAlloc":268768720, "HeapSys":934969344, "HeapIdle":660619264, "HeapInuse":274350080,"HeapReleased":433594368, "HeapObjects":536335,"GCSys":30900224, "LastGC":1631582673155988058,"PauseTotalNs":146660601559, "PauseNs":[], "NumGC":19299} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:25:21 PDT 2021 RSS = 3953144 2021-09-13 18:25:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.40000, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 574063809, "lss_used_space": 3250318468, "lss_fragmentation": 82, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 219179237376, "bytes_incoming": 71502511416, "write_amp_avg": 2.00, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 153900612, "recovery_used_space": 706946691, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123521847296, "recovery_lss_frag": 78, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }2021-09-13T18:25:45.395-07:00 [Info] memstats {"Alloc":270512872, "TotalAlloc":3681940951200, "Sys":976283896, "Lookups":0, "Mallocs":241044141,"Frees":240488723, "HeapAlloc":270512872, "HeapSys":934969344, "HeapIdle":659759104, "HeapInuse":275210240,"HeapReleased":433594368, "HeapObjects":555418,"GCSys":30900224, "LastGC":1631582673155988058,"PauseTotalNs":146660601559, "PauseNs":[], "NumGC":19299} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:26:15.422-07:00 [Info] memstats {"Alloc":272209320, "TotalAlloc":3681942647648, "Sys":976283896, "Lookups":0, "Mallocs":241063095,"Frees":240488732, "HeapAlloc":272209320, "HeapSys":934969344, "HeapIdle":658726912, "HeapInuse":276242432,"HeapReleased":433594368, "HeapObjects":574363,"GCSys":30900224, "LastGC":1631582673155988058,"PauseTotalNs":146660601559, "PauseNs":[], "NumGC":19299} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:26:21 PDT 2021 RSS = 3953144 2021-09-13T18:26:45.460-07:00 [Info] memstats {"Alloc":268004504, "TotalAlloc":3681944313480, "Sys":976283896, "Lookups":0, "Mallocs":241082070,"Frees":240552884, "HeapAlloc":268004504, "HeapSys":934969344, "HeapIdle":660455424, "HeapInuse":274513920,"HeapReleased":433594368, "HeapObjects":529186,"GCSys":30900224, "LastGC":1631582793277894728,"PauseTotalNs":146661010366, "PauseNs":[408807], "NumGC":19300} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:27:15.486-07:00 [Info] memstats {"Alloc":269666480, "TotalAlloc":3681945975456, "Sys":976283896, "Lookups":0, "Mallocs":241101029,"Frees":240552893, "HeapAlloc":269666480, "HeapSys":934969344, "HeapIdle":659652608, "HeapInuse":275316736,"HeapReleased":433594368, "HeapObjects":548136,"GCSys":30900224, "LastGC":1631582793277894728,"PauseTotalNs":146661010366, "PauseNs":[], "NumGC":19300} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:27:21 PDT 2021 RSS = 3953144 2021-09-13T18:27:45.518-07:00 [Info] memstats {"Alloc":271345512, "TotalAlloc":3681947654488, "Sys":976283896, "Lookups":0, "Mallocs":241120007,"Frees":240552902, "HeapAlloc":271345512, "HeapSys":934969344, "HeapIdle":658964480, "HeapInuse":276004864,"HeapReleased":433594368, "HeapObjects":567105,"GCSys":30900224, "LastGC":1631582793277894728,"PauseTotalNs":146661010366, "PauseNs":[], "NumGC":19300} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:28:15.545-07:00 [Info] memstats {"Alloc":273032568, "TotalAlloc":3681949341544, "Sys":976283896, "Lookups":0, "Mallocs":241138962,"Frees":240552911, "HeapAlloc":273032568, "HeapSys":934969344, "HeapIdle":657965056, "HeapInuse":277004288,"HeapReleased":433594368, "HeapObjects":586051,"GCSys":30900224, "LastGC":1631582793277894728,"PauseTotalNs":146661010366, "PauseNs":[], "NumGC":19300} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:28:22 PDT 2021 RSS = 3953144 2021-09-13T18:28:45.579-07:00 [Info] memstats {"Alloc":268964392, "TotalAlloc":3681951036240, "Sys":976283896, "Lookups":0, "Mallocs":241157931,"Frees":240616917, "HeapAlloc":268964392, "HeapSys":934969344, "HeapIdle":659390464, "HeapInuse":275578880,"HeapReleased":433594368, "HeapObjects":541014,"GCSys":30900224, "LastGC":1631582913420540896,"PauseTotalNs":146661456516, "PauseNs":[446150], "NumGC":19301} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:29:15.617-07:00 [Info] memstats {"Alloc":270642864, "TotalAlloc":3681952714712, "Sys":976283896, "Lookups":0, "Mallocs":241176904,"Frees":240616926, "HeapAlloc":270642864, "HeapSys":934969344, "HeapIdle":658751488, "HeapInuse":276217856,"HeapReleased":433594368, "HeapObjects":559978,"GCSys":30900224, "LastGC":1631582913420540896,"PauseTotalNs":146661456516, "PauseNs":[], "NumGC":19301} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 Mon Sep 13 18:29:22 PDT 2021 RSS = 3953144 2021-09-13T18:29:45.647-07:00 [Info] memstats {"Alloc":272330056, "TotalAlloc":3681954401904, "Sys":976283896, "Lookups":0, "Mallocs":241195877,"Frees":240616935, "HeapAlloc":272330056, "HeapSys":934969344, "HeapIdle":657973248, "HeapInuse":276996096,"HeapReleased":433594368, "HeapObjects":578942,"GCSys":30900224, "LastGC":1631582913420540896,"PauseTotalNs":146661456516, "PauseNs":[], "NumGC":19301} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13T18:30:15.673-07:00 [Info] memstats {"Alloc":276142216, "TotalAlloc":3681958214064, "Sys":976283896, "Lookups":0, "Mallocs":241218742,"Frees":240617635, "HeapAlloc":276142216, "HeapSys":934969344, "HeapIdle":654557184, "HeapInuse":280412160,"HeapReleased":433594368, "HeapObjects":601107,"GCSys":30900224, "LastGC":1631582913420540896,"PauseTotalNs":146661456516, "PauseNs":[], "NumGC":19301} Allocated: 3738833704, active: 3740188672, metadata: 47935152 (n_thp 0), resident: 3858661376, mapped: 3875192832, retained: 15561211904 2021-09-13 18:30:22 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 243, "num_active_sctxs": 87, "num_free_sctxs": 156, "num_trim_sctxs": 11, "num_wctxs": 294, "num_free_wctxs": 237, "num_active_wctxs": 57, "num_trim_wctxs": 1387, "num_readers": 68, "num_writers": 118, "cpu": 8.41000, "total_core": 16, "threads": 58, "assigned_quota": 1516282026, "current_quota": 1516282026, "memory_resident": 0, "memory_jemalloc": 3858661376, "memory_jemalloc_allocated": 3738833704, "memory_in_use": 3858661376, "memory_stats_size_total": 3293097365, "memory_stats_size_global": 1344605024, "memory_stats_size_page": 1211138832, "memory_stats_size_index": 133809824, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 204720, "reclaim_pending": 0, "buf_memused": 1948148709, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2050, "items_count": 2080, "total_records": 16432, "total_ops": 1090842, "compacts": 3145449, "purges": 1522, "lss_data_size": 574063809, "lss_used_space": 3250318468, "lss_fragmentation": 82, "lss_num_reads": 1134918, "lss_read_bs": 215978630730, "lss_blk_read_bs": 220619546624, "rlss_num_reads": 0, "lss_rdr_reads_bs": 0, "lss_blk_rdr_reads_bs": 0, "bytes_written": 219179237376, "bytes_incoming": 71502511416, "write_amp_avg": 2.00, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 253 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 16432, "resident_ratio": 100.00000, "cleaner_num_reads": 1134918, "cleaner_read_bs": 215978630730, "cleaner_blk_read_bs": 220619546624, "recovery_num_lssctx": 5, "recovery_num_sctxs": 31, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 11, "recovery_num_trim_sctxs": 11, "recovery_buf_memused": 5242880, "recovery_data_size": 153900612, "recovery_used_space": 706946691, "recovery_lss_blk_read_bs": 126147796992, "recovery_lss_read_bs": 122555199076, "recovery_lss__num_reads": 951541, "recovery_bytes_written": 123521847296, "recovery_lss_frag": 78, "recovery_cleaner_num_reads": 951541, "recovery_cleaner_read_bs": 122555199076, "recovery_cleaner_blk_read_bs": 126147796992, "mvcc_purge_ratio": 7.90000 }Mon Sep 13 18:30:22 PDT 2021 RSS = 3953144 Time to kill! error: process ID list syntax error Usage: ps [options] Try 'ps --help ' or 'ps --help ' for additional help text. For more details see ps(1). Mon Sep 13 18:31:22 PDT 2021 RSS = ./run: line 24: 60854 Killed ./gsi $1 false 2>&1 Initializing write barrier = 8000 2021-09-13 18:31:27 [gsi-test] [] Total number of index instances 10 2021-09-13 18:31:27 [gsi-test] [] Local MemQuota: -------------- GSI-Plasma Test Config ------------------ {"RunTime":"3h","MemQuota":"5gb","QuotaTuner":true,"QuotaTunerMin":20,"QuotaTunerMax":50,"Verify":true,"VerifyTime":"1m","RecoveryInterval":"210m","SuppressLogs":false,"StopAndVerify":true,"StopAndVerifyTime":"5m","MemoryEvaluationMinimum":16777216,"MemoryEvaluationThreshold":1.6,"ActiveKeysConsistencyThreshold":0.85} -------------- GSI-Plasma Default Config ------------------ {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":null,"IdxRunTime":"120m","Load":0,"Workload":null,"Shared":true,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} -------------- GSI-Plasma Index Config ------------------ {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_0","IdxRunTime":"120m","Load":0,"Workload":"config/workload/01_ru_workload.json","Shared":false,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_1","IdxRunTime":"120m","Load":0,"Workload":"config/workload/01_ru_workload.json","Shared":false,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_2","IdxRunTime":"120m","Load":0,"Workload":"config/workload/01_ru_workload.json","Shared":true,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_3","IdxRunTime":"120m","Load":0,"Workload":"config/workload/01_ru_workload.json","Shared":true,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_4","IdxRunTime":"120m","Load":0,"Workload":"config/workload/02_rh_workload.json","Shared":false,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_5","IdxRunTime":"120m","Load":0,"Workload":"config/workload/02_rh_workload.json","Shared":true,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_6","IdxRunTime":"120m","Load":0,"Workload":"config/workload/02_rh_workload.json","Shared":true,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_7","IdxRunTime":"120m","Load":0,"Workload":"config/workload/03_ruc_workload.json","Shared":false,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1000},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_8","IdxRunTime":"120m","Load":0,"Workload":"config/workload/03_ruc_workload.json","Shared":true,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1000},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} {"NumKeys":200,"NumWorkers":16,"KeyDistribution":"","KeySize":[{"Min":65536,"Max":65536}],"SwitchGeneration":"24h","UseZipf":null,"Tracker":null,"ResidentRatio":0.2,"MinValueLen":64,"MaxValueLen":64,"IdxPath":"/data/gsi_testdir/devel_9","IdxRunTime":"120m","Load":0,"Workload":"config/workload/03_ruc_workload.json","Shared":true,"Snapshot":{"Frequency":"20ms","CountMax":500},"RecoveryPoint":{"Count":3,"Frequency":"10m","RollbackFrequency":"60m"},"PersistAll":"5m","Delete":{"RangeMin":8,"RangeMax":32,"Frequency":"60m","AllFrequency":"90m"},"LastInserted":{"Count":1000},"ScanQuery":[{"RangeMin":1,"RangeMax":3,"Priority":1}]} 2021-09-13 18:31:27 [gsi-test] [] Set global quota: 5368709120 2021-09-13 18:31:27 [gsi-test] [] The runtime is 10800 seconds in 3h0m0s. 2021-09-13 18:31:27 [gsi-test] [] Start Time: 2021-09-13 18:31:27.93150024 -0700 PDT m=+0.028413553 End Time: 2021-09-13 21:31:27.93150024 -0700 PDT m=+10800.028413553 2021-09-13 18:31:27 [gsi-test] [0] Init index instance %d with recovery 2021-09-13 18:31:27 [plasma] [i] Start shard recovery from /data/gsi_testdir/shards2021-09-13 18:31:27 [plasma] [i] : Recover shard shard1 from metadata completed. 2021-09-13 18:31:27 [plasma] [i] : Recover shard shard2 from metadata completed. 2021-09-13 18:31:27 [plasma] [i] : Recover shard shard3 from metadata completed. 2021-09-13 18:31:27 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner initialized 2021-09-13 18:31:27 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_0_0/docIndex 2021-09-13 18:31:27 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCleaner initialized for recovery 2021-09-13 18:31:27 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_0_0/docIndex/recovery 2021-09-13 18:31:27 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Map plasma instance (/data/gsi_testdir/devel_0_0/docIndex) to LSS (/data/gsi_testdir/devel_0_0/docIndex) and RecoveryLSS (/data/gsi_testdir/devel_0_0/docIndex/recovery) 2021-09-13 18:31:27 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Starting recovery. Recovery log [/data/gsi_testdir/devel_0_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_0_0/docIndex], Shared [false] 2021-09-13 18:31:27 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoverFromHeaderReplay: Begin recovering from recovery log, headOffset [1271996836] tailOffset [1360687104] 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Done recovering from recovery log, replayOffset [7753502720] took [198.951826ms] 2021-09-13 18:31:28 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : recoverFromDataReplay: Begin recovering from data log, headOffset [7512993955] tailOffset [7753506816] replayOffset [7753502720] 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Done recovering from data and recovery log, took [199.998189ms] 2021-09-13 18:31:28 [plasma] [i] Plasma: /data/gsi_testdir/devel_0_0/docIndex: Repair 0 added 0 deleted 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Done recovery. Recovery log [/data/gsi_testdir/devel_0_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_0_0/docIndex], Shared [false]. Built [1] plasmas, took [205.570331ms] 2021-09-13 18:31:28 [plasma] [i] Plasma: doInit: data UsedSpace 240512861 recovery UsedSpace 88358136 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Assign plasmaId 1 to instance /data/gsi_testdir/devel_0_0/docIndex 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Add instance /data/gsi_testdir/devel_0_0/docIndex to Shard /data/gsi_testdir/shards/shard1 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Add instance /data/gsi_testdir/devel_0_0/docIndex to LSS /data/gsi_testdir/devel_0_0/docIndex 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper started 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Instance added to swapper : /data/gsi_testdir/shards/shard1 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : instance /data/gsi_testdir/devel_0_0/docIndex started 2021-09-13 18:31:28 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 18:31:28 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 18:31:28 [gsi-test] [1] Init index instance %d with recovery 2021-09-13 18:31:28 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner initialized 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_1_0/docIndex 2021-09-13 18:31:28 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCleaner initialized for recovery 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_1_0/docIndex/recovery 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Map plasma instance (/data/gsi_testdir/devel_1_0/docIndex) to LSS (/data/gsi_testdir/devel_1_0/docIndex) and RecoveryLSS (/data/gsi_testdir/devel_1_0/docIndex/recovery) 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Shard.doRecovery: Starting recovery. Recovery log [/data/gsi_testdir/devel_1_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_1_0/docIndex], Shared [false] 2021-09-13 18:31:28 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoverFromHeaderReplay: Begin recovering from recovery log, headOffset [4088308132] tailOffset [4173373440] 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Shard.doRecovery: Done recovering from recovery log, replayOffset [8484405248] took [191.010901ms] 2021-09-13 18:31:28 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : recoverFromDataReplay: Begin recovering from data log, headOffset [8253207321] tailOffset [8484409344] replayOffset [8484405248] 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Shard.doRecovery: Done recovering from data and recovery log, took [191.399468ms] 2021-09-13 18:31:28 [plasma] [i] Plasma: /data/gsi_testdir/devel_1_0/docIndex: Repair 0 added 0 deleted 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Shard.doRecovery: Done recovery. Recovery log [/data/gsi_testdir/devel_1_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_1_0/docIndex], Shared [false]. Built [1] plasmas, took [197.389801ms] 2021-09-13 18:31:28 [plasma] [i] Plasma: doInit: data UsedSpace 231202023 recovery UsedSpace 84739362 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Assign plasmaId 1 to instance /data/gsi_testdir/devel_1_0/docIndex 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Add instance /data/gsi_testdir/devel_1_0/docIndex to Shard /data/gsi_testdir/shards/shard2 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Add instance /data/gsi_testdir/devel_1_0/docIndex to LSS /data/gsi_testdir/devel_1_0/docIndex 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper started 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Instance added to swapper : /data/gsi_testdir/shards/shard2 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : instance /data/gsi_testdir/devel_1_0/docIndex started 2021-09-13 18:31:28 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 18:31:28 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 18:31:28 [gsi-test] [2] Init index instance %d with recovery 2021-09-13 18:31:28 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner initialized 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : LSSCtx Created Successfully. Path=/data/gsi_testdir/shards/shard3/data 2021-09-13 18:31:28 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCleaner initialized for recovery 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : LSSCtx Created Successfully. Path=/data/gsi_testdir/shards/shard3/data/recovery 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Map plasma instance (/data/gsi_testdir/devel_2_0/docIndex) to LSS (/data/gsi_testdir/shards/shard3/data) and RecoveryLSS (/data/gsi_testdir/shards/shard3/data/recovery) 2021-09-13 18:31:28 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Shard.doRecovery: Starting recovery. Recovery log [/data/gsi_testdir/shards/shard3/data/recovery], Data log [/data/gsi_testdir/shards/shard3/data], Shared [true] 2021-09-13 18:31:28 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoverFromHeaderReplay: Begin recovering from recovery log, headOffset [6739411190] tailOffset [7060496384] 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Shard.doRecovery: Done recovering from recovery log, replayOffset [42788876288] took [848.49761ms] 2021-09-13 18:31:29 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : recoverFromDataReplay: Begin recovering from data log, headOffset [41257970169] tailOffset [42825854976] replayOffset [42788876288] 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Shard.doRecovery: Done recovering from data and recovery log, took [1.210354478s] 2021-09-13 18:31:29 [plasma] [i] Plasma: /data/gsi_testdir/devel_8_0/docIndex: Repair 0 added 0 deleted 2021-09-13 18:31:29 [plasma] [i] Plasma: /data/gsi_testdir/devel_2_0/docIndex: Repair 0 added 0 deleted 2021-09-13 18:31:29 [plasma] [i] Plasma: /data/gsi_testdir/devel_3_0/docIndex: Repair 0 added 0 deleted 2021-09-13 18:31:29 [plasma] [i] Plasma: /data/gsi_testdir/devel_6_0/docIndex: Repair 0 added 0 deleted 2021-09-13 18:31:29 [plasma] [i] Plasma: /data/gsi_testdir/devel_5_0/docIndex: Repair 0 added 0 deleted 2021-09-13 18:31:29 [plasma] [i] Plasma: /data/gsi_testdir/devel_9_0/docIndex: Repair 0 added 0 deleted 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Shard.doRecovery: Done recovery. Recovery log [/data/gsi_testdir/shards/shard3/data/recovery], Data log [/data/gsi_testdir/shards/shard3/data], Shared [true]. Built [6] plasmas, took [1.239708935s] 2021-09-13 18:31:29 [plasma] [i] Plasma: doInit: data UsedSpace 216954910 recovery UsedSpace 45785678 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Assign plasmaId 1 to instance /data/gsi_testdir/devel_2_0/docIndex 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_2_0/docIndex to Shard /data/gsi_testdir/shards/shard3 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_2_0/docIndex to LSS /data/gsi_testdir/shards/shard3/data 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper started 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Instance added to swapper : /data/gsi_testdir/shards/shard3 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : instance /data/gsi_testdir/devel_2_0/docIndex started 2021-09-13 18:31:29 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 18:31:29 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 18:31:29 [gsi-test] [3] Init index instance %d with recovery 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Map plasma instance (/data/gsi_testdir/devel_3_0/docIndex) to LSS (/data/gsi_testdir/shards/shard3/data) and RecoveryLSS (/data/gsi_testdir/shards/shard3/data/recovery) 2021-09-13 18:31:29 [plasma] [i] Plasma: doInit: data UsedSpace 265525264 recovery UsedSpace 41785162 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Assign plasmaId 2 to instance /data/gsi_testdir/devel_3_0/docIndex 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_3_0/docIndex to Shard /data/gsi_testdir/shards/shard3 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_3_0/docIndex to LSS /data/gsi_testdir/shards/shard3/data 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Instance added to swapper : /data/gsi_testdir/shards/shard3 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : instance /data/gsi_testdir/devel_3_0/docIndex started 2021-09-13 18:31:29 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 18:31:29 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 18:31:29 [gsi-test] [4] Init index instance %d with recovery 2021-09-13 18:31:29 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner initialized 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_4_0/docIndex 2021-09-13 18:31:29 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCleaner initialized for recovery 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_4_0/docIndex/recovery 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Map plasma instance (/data/gsi_testdir/devel_4_0/docIndex) to LSS (/data/gsi_testdir/devel_4_0/docIndex) and RecoveryLSS (/data/gsi_testdir/devel_4_0/docIndex/recovery) 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Starting recovery. Recovery log [/data/gsi_testdir/devel_4_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_4_0/docIndex], Shared [false] 2021-09-13 18:31:29 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoverFromHeaderReplay: Begin recovering from recovery log, headOffset [59025895564] tailOffset [59140845568] 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Done recovering from recovery log, replayOffset [27592417280] took [235.153688ms] 2021-09-13 18:31:29 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : recoverFromDataReplay: Begin recovering from data log, headOffset [27305754220] tailOffset [27592421376] replayOffset [27592417280] 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Done recovering from data and recovery log, took [235.381091ms] 2021-09-13 18:31:29 [plasma] [i] Plasma: /data/gsi_testdir/devel_4_0/docIndex: Repair 0 added 0 deleted 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Done recovery. Recovery log [/data/gsi_testdir/devel_4_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_4_0/docIndex], Shared [false]. Built [1] plasmas, took [239.338083ms] 2021-09-13 18:31:29 [plasma] [i] Plasma: doInit: data UsedSpace 286667156 recovery UsedSpace 112541466 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Assign plasmaId 2 to instance /data/gsi_testdir/devel_4_0/docIndex 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Add instance /data/gsi_testdir/devel_4_0/docIndex to Shard /data/gsi_testdir/shards/shard1 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Add instance /data/gsi_testdir/devel_4_0/docIndex to LSS /data/gsi_testdir/devel_4_0/docIndex 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Instance added to swapper : /data/gsi_testdir/shards/shard1 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : instance /data/gsi_testdir/devel_4_0/docIndex started 2021-09-13 18:31:29 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 18:31:29 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 18:31:29 [gsi-test] [5] Init index instance %d with recovery 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Map plasma instance (/data/gsi_testdir/devel_5_0/docIndex) to LSS (/data/gsi_testdir/shards/shard3/data) and RecoveryLSS (/data/gsi_testdir/shards/shard3/data/recovery) 2021-09-13 18:31:29 [plasma] [i] Plasma: doInit: data UsedSpace 258900171 recovery UsedSpace 68864346 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Assign plasmaId 3 to instance /data/gsi_testdir/devel_5_0/docIndex 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_5_0/docIndex to Shard /data/gsi_testdir/shards/shard3 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_5_0/docIndex to LSS /data/gsi_testdir/shards/shard3/data 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Instance added to swapper : /data/gsi_testdir/shards/shard3 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : instance /data/gsi_testdir/devel_5_0/docIndex started 2021-09-13 18:31:29 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 18:31:29 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 18:31:29 [gsi-test] [6] Init index instance %d with recovery 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Map plasma instance (/data/gsi_testdir/devel_6_0/docIndex) to LSS (/data/gsi_testdir/shards/shard3/data) and RecoveryLSS (/data/gsi_testdir/shards/shard3/data/recovery) 2021-09-13 18:31:29 [plasma] [i] Plasma: doInit: data UsedSpace 257614107 recovery UsedSpace 54371800 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Assign plasmaId 4 to instance /data/gsi_testdir/devel_6_0/docIndex 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_6_0/docIndex to Shard /data/gsi_testdir/shards/shard3 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_6_0/docIndex to LSS /data/gsi_testdir/shards/shard3/data 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Instance added to swapper : /data/gsi_testdir/shards/shard3 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : instance /data/gsi_testdir/devel_6_0/docIndex started 2021-09-13 18:31:29 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 18:31:29 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 18:31:29 [gsi-test] [7] Init index instance %d with recovery 2021-09-13 18:31:29 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner initialized 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_7_0/docIndex 2021-09-13 18:31:29 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCleaner initialized for recovery 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : LSSCtx Created Successfully. Path=/data/gsi_testdir/devel_7_0/docIndex/recovery 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Map plasma instance (/data/gsi_testdir/devel_7_0/docIndex) to LSS (/data/gsi_testdir/devel_7_0/docIndex) and RecoveryLSS (/data/gsi_testdir/devel_7_0/docIndex/recovery) 2021-09-13 18:31:29 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Starting recovery. Recovery log [/data/gsi_testdir/devel_7_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_7_0/docIndex], Shared [false] 2021-09-13 18:31:29 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoverFromHeaderReplay: Begin recovering from recovery log, headOffset [51687645604] tailOffset [51786477568] 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Done recovering from recovery log, replayOffset [9017708544] took [148.893934ms] 2021-09-13 18:31:30 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : recoverFromDataReplay: Begin recovering from data log, headOffset [8762877772] tailOffset [9017712640] replayOffset [9017708544] 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Done recovering from data and recovery log, took [149.138556ms] 2021-09-13 18:31:30 [plasma] [i] Plasma: /data/gsi_testdir/devel_7_0/docIndex: Repair 0 added 0 deleted 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Shard.doRecovery: Done recovery. Recovery log [/data/gsi_testdir/devel_7_0/docIndex/recovery], Data log [/data/gsi_testdir/devel_7_0/docIndex], Shared [false]. Built [1] plasmas, took [152.762173ms] 2021-09-13 18:31:30 [plasma] [i] Plasma: doInit: data UsedSpace 254834868 recovery UsedSpace 98451936 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Assign plasmaId 3 to instance /data/gsi_testdir/devel_7_0/docIndex 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Add instance /data/gsi_testdir/devel_7_0/docIndex to Shard /data/gsi_testdir/shards/shard1 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Add instance /data/gsi_testdir/devel_7_0/docIndex to LSS /data/gsi_testdir/devel_7_0/docIndex 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Instance added to swapper : /data/gsi_testdir/shards/shard1 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : instance /data/gsi_testdir/devel_7_0/docIndex started 2021-09-13 18:31:30 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 18:31:30 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 18:31:30 [gsi-test] [8] Init index instance %d with recovery 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Map plasma instance (/data/gsi_testdir/devel_8_0/docIndex) to LSS (/data/gsi_testdir/shards/shard3/data) and RecoveryLSS (/data/gsi_testdir/shards/shard3/data/recovery) 2021-09-13 18:31:30 [plasma] [i] Plasma: doInit: data UsedSpace 254821971 recovery UsedSpace 61196012 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Assign plasmaId 5 to instance /data/gsi_testdir/devel_8_0/docIndex 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_8_0/docIndex to Shard /data/gsi_testdir/shards/shard3 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_8_0/docIndex to LSS /data/gsi_testdir/shards/shard3/data 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Instance added to swapper : /data/gsi_testdir/shards/shard3 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : instance /data/gsi_testdir/devel_8_0/docIndex started 2021-09-13 18:31:30 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 18:31:30 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 18:31:30 [gsi-test] [9] Init index instance %d with recovery 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Map plasma instance (/data/gsi_testdir/devel_9_0/docIndex) to LSS (/data/gsi_testdir/shards/shard3/data) and RecoveryLSS (/data/gsi_testdir/shards/shard3/data/recovery) 2021-09-13 18:31:30 [plasma] [i] Plasma: doInit: data UsedSpace 263748892 recovery UsedSpace 31878802 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Assign plasmaId 6 to instance /data/gsi_testdir/devel_9_0/docIndex 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_9_0/docIndex to Shard /data/gsi_testdir/shards/shard3 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Add instance /data/gsi_testdir/devel_9_0/docIndex to LSS /data/gsi_testdir/shards/shard3/data 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Instance added to swapper : /data/gsi_testdir/shards/shard3 2021-09-13 18:31:30 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : instance /data/gsi_testdir/devel_9_0/docIndex started 2021-09-13 18:31:30 [plasma] [i] Plasma: Plasma.mvccPurgerDaemon: Starting MVCC Purger 2021-09-13 18:31:30 [plasma] [i] Plasma: Plasma.monitor: Starting monitor 2021-09-13 18:31:30 [gsi-test] [9] Instance... START 2021-09-13 18:31:30 [gsi-test] [9] Starting job for index instance 2021-09-13 18:31:30 [gsi-test] [9] CreateSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [4] Instance... START 2021-09-13 18:31:30 [gsi-test] [9] DeleteSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [4] Starting job for index instance 2021-09-13 18:31:30 [gsi-test] [0] Instance... START 2021-09-13 18:31:30 [gsi-test] [0] Starting job for index instance 2021-09-13 18:31:30 [gsi-test] [4] CreateSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [0] CreateSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [5] Instance... START 2021-09-13 18:31:30 [gsi-test] [5] Starting job for index instance 2021-09-13 18:31:30 [gsi-test] [5] CreateSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [9:5] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5] DeleteSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [9:5] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:5] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [0] DeleteSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [4] DeleteSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [0:15] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [0:15] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2] Instance... START 2021-09-13 18:31:30 [gsi-test] [0:15] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2] Starting job for index instance 2021-09-13 18:31:30 [gsi-test] [2] CreateSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [2] DeleteSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [4:15] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:15] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [4:15] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [5:15] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5:15] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:15] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [0:7] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [0:7] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:7] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:15] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:15] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2:15] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [5:2] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5:2] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:2] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:0] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:0] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2:0] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [0:11] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [0:11] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:11] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4:6] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:6] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [4:6] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [9:1] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:6] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:1] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:0] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:1] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [0:0] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:0] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [0:9] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [0:8] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:6] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:8] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:6] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [0:8] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [0:13] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [0:13] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:13] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:2] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:2] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2:2] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1] Instance... START 2021-09-13 18:31:30 [gsi-test] [9:7] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:7] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:7] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:3] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:3] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:12] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [0:12] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:12] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [5:5] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:15] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:1] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:1] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2:1] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:3] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6] Instance... START 2021-09-13 18:31:30 [gsi-test] [6] Starting job for index instance 2021-09-13 18:31:30 [gsi-test] [5:3] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5:3] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:3] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [0:9] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:9] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1] Starting job for index instance 2021-09-13 18:31:30 [gsi-test] [7] Instance... START 2021-09-13 18:31:30 [gsi-test] [7] Starting job for index instance 2021-09-13 18:31:30 [gsi-test] [3] Instance... START 2021-09-13 18:31:30 [gsi-test] [5:5] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:5] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [5:6] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5:6] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:6] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [0:1] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [0:1] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:1] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [5:4] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5:4] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:4] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4:8] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [0:10] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:8] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:10] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:11] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [0:10] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [9:11] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:11] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4:0] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:0] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [4:0] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4:1] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:1] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [4:1] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [9:4] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:4] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:4] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [0:2] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [0:2] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:2] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4:8] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:5] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:5] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2:5] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:6] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:6] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2:6] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [9:8] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:8] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:8] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:7] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:7] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2:7] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4:7] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:7] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [4:7] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [9:2] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:2] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:2] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [9:3] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:3] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:3] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4:9] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:9] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [4:9] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4:10] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:10] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [4:10] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4:11] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:11] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [4:11] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4:12] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:12] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [4:12] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4:13] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:13] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [4:13] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4:14] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:14] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [4:14] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4:2] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:2] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [4:2] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6] CreateSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [5:7] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5:7] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:7] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [5:9] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5:9] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:9] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [0:14] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [0:14] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:14] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [5:12] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5:12] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:12] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [5:13] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5:13] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:13] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [5:14] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5:14] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:14] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [5:11] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5:11] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:11] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7] CreateSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [6] DeleteSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [6:15] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [6:15] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:15] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [9:9] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:9] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:9] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [9:10] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:10] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:10] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:8] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:8] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2:8] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [9:12] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:12] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:12] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:9] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:9] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2:9] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:10] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:10] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2:10] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:11] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:11] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2:11] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [0:4] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [0:4] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:4] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [0:5] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [0:5] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:5] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [0:6] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [0:6] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:6] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [9:0] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:0] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:0] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [9:15] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:15] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [9:13] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:13] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:13] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [9:14] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [9:14] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [9:14] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6:1] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [6:1] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:1] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6:2] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [6:2] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:2] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6:3] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [6:3] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:3] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6:4] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [6:4] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:4] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6:5] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [6:5] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:5] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6:6] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [6:6] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:6] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6:7] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [6:7] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:7] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6:13] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [6:13] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:13] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6:12] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [6:14] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [6:14] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:14] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6:12] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:12] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3] Starting job for index instance 2021-09-13 18:31:30 [gsi-test] [9] CreateRecoveryPoint() - START 2021-09-13 18:31:30 [gsi-test] [6:0] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8] Instance... START 2021-09-13 18:31:30 [gsi-test] [8] Starting job for index instance 2021-09-13 18:31:30 [gsi-test] [8] CreateSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [8] DeleteSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [8:15] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:15] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:15] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8:0] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:0] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:0] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8:7] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:7] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:7] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8:11] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:11] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:11] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8:12] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:12] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:12] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4:3] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:3] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [4:3] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8:3] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:3] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:3] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8:13] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:13] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:13] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8:1] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:1] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:1] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8:14] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:14] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:14] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8:2] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:2] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:2] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8:9] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:9] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:9] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8:8] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:8] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:5] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:5] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:6] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:6] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:8] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8:6] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8:5] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8:4] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:10] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [8:4] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:10] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [8:4] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8:10] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [8] CreateRecoveryPoint() - START 2021-09-13 18:31:30 [gsi-test] [8] Rollback() - START 2021-09-13 18:31:30 [gsi-test] [8] Generation() - START 2021-09-13 18:31:30 [gsi-test] [5:8] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5:8] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:8] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [5:10] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5:10] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:10] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1] CreateSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [1] DeleteSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [1:15] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:15] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:15] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1:6] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:6] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:6] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1:10] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:10] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:10] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1:7] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:7] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:7] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1:8] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:8] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:8] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1:12] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:12] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:12] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1:2] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:2] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:2] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1:9] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:9] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:9] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1:0] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:0] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:0] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1:13] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:13] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:13] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:14] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:14] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2:14] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1:11] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:11] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:11] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1:3] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:3] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:3] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:13] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:13] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2:13] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:12] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:12] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2:12] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1:4] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:4] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:4] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1:14] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:14] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:14] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1:1] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:1] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:1] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1:5] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [1:5] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [1:5] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4:5] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:5] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [4:5] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6:10] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [6:10] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:10] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [0:3] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [0:3] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [0:3] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2:4] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [2:4] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [2:4] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [2] CreateRecoveryPoint() - START 2021-09-13 18:31:30 [gsi-test] [2] Rollback() - START 2021-09-13 18:31:30 [gsi-test] [2] Generation() - START 2021-09-13 18:31:30 [gsi-test] [6:9] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [6:9] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:9] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6:11] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [6:11] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:11] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [5:1] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5:1] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:1] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [1] CreateRecoveryPoint() - START 2021-09-13 18:31:30 [gsi-test] [1] Rollback() - START 2021-09-13 18:31:30 [gsi-test] [1] Generation() - START 2021-09-13 18:31:30 [gsi-test] [3] CreateSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [3] DeleteSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [3:15] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:15] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:15] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3:6] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:6] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:6] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6:8] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [6:8] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:8] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6] CreateRecoveryPoint() - START 2021-09-13 18:31:30 [gsi-test] [6] Rollback() - START 2021-09-13 18:31:30 [gsi-test] [6] Generation() - START 2021-09-13 18:31:30 [gsi-test] [3:0] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:0] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:0] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3:10] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:10] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:10] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3:8] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:8] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:8] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3:7] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:7] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:7] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3:12] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:12] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:12] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3:13] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:13] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:13] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3:14] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:14] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:14] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3:9] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:9] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:9] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3:3] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:3] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:3] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3:1] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:1] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:1] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3:2] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:2] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:2] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3:11] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:11] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:11] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3:5] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:5] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:5] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3:4] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [3:4] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [3:4] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [3] CreateRecoveryPoint() - START 2021-09-13 18:31:30 [gsi-test] [3] Rollback() - START 2021-09-13 18:31:30 [gsi-test] [3] Generation() - START 2021-09-13 18:31:30 [gsi-test] [9] Rollback() - START 2021-09-13 18:31:30 [gsi-test] [9] Generation() - START 2021-09-13 18:31:30 [gsi-test] [0] CreateRecoveryPoint() - START 2021-09-13 18:31:30 [gsi-test] [0] Rollback() - START 2021-09-13 18:31:30 [gsi-test] [0] Generation() - START 2021-09-13 18:31:30 [gsi-test] [5:0] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [5:0] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [5:0] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [5] CreateRecoveryPoint() - START 2021-09-13 18:31:30 [gsi-test] [5] Rollback() - START 2021-09-13 18:31:30 [gsi-test] [5] Generation() - START 2021-09-13 18:31:30 [gsi-test] [7] DeleteSnapshot() - START 2021-09-13 18:31:30 [gsi-test] [7:15] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:15] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:15] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7:6] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:6] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:6] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7:10] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:10] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:10] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7:12] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:12] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:12] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7:13] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:13] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:13] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7:14] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:14] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:14] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7:7] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:7] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:7] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7:8] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:8] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:8] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7:9] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:9] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:9] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7:2] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:2] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:2] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [6:0] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [6:0] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7:0] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:0] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:0] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7:4] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:4] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:4] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7:11] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:11] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:11] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7:3] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:3] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:3] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7:1] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:1] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:1] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7:5] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [7:5] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [7:5] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [7] CreateRecoveryPoint() - START 2021-09-13 18:31:30 [gsi-test] [7] Rollback() - START 2021-09-13 18:31:30 [gsi-test] [7] Generation() - START 2021-09-13 18:31:30 [gsi-test] [4:4] Worker.Run() - START 2021-09-13 18:31:30 [gsi-test] [4:4] Worker.Run() - LOAD 2021-09-13 18:31:30 [gsi-test] [4:4] Worker.Run() - WORKLOAD 2021-09-13 18:31:30 [gsi-test] [4] CreateRecoveryPoint() - START 2021-09-13 18:31:30 [gsi-test] [4] Rollback() - START 2021-09-13 18:31:30 [gsi-test] [4] Generation() - START 2021-09-13 18:31:34 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 70491 KB (expected = 10240 KB), wCtxCnt = 18, objCnt 125, changed reclaimList flush threshold from 50 to 0, changed reclaimSize flush threshold from 10240 KB to 455 KB. 2021-09-13 18:31:34 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 79327 KB (expected = 10240 KB), wCtxCnt = 18, objCnt 146, changed reclaimList flush threshold from 50 to 0, changed reclaimSize flush threshold from 10240 KB to 455 KB. 2021-09-13 18:31:38 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 41043 KB (expected = 10240 KB), wCtxCnt = 13, objCnt 97, changed reclaimList flush threshold from 50 to 1, changed reclaimSize flush threshold from 10240 KB to 630 KB. 2021-09-13 18:31:40 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 47380 KB (expected = 10240 KB), wCtxCnt = 12, objCnt 94, changed reclaimList flush threshold from 50 to 1, changed reclaimSize flush threshold from 10240 KB to 682 KB. 2021-09-13 18:31:42 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 56279 KB (expected = 10240 KB), wCtxCnt = 11, objCnt 99, changed reclaimList flush threshold from 50 to 1, changed reclaimSize flush threshold from 10240 KB to 744 KB. 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : all deamons started 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : all deamons started 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : all deamons started 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : all deamons started 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : all deamons started 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : all deamons started 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : all deamons started 2021-09-13 18:31:50 [gsi-test] [] Stop and verify... START 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:31:50 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:31:50 [gsi-test] [] GSI-Plasma Current Time 2021-09-13 18:31:50.079887857 -0700 PDT m=+22.176802119 2021-09-13 18:31:50 [gsi-test] [] MemoryStats... START 2021-09-13 18:31:50 [gsi-test] [] ShortStats... START 2021-09-13 18:31:50 [gsi-test] [] FullStats... START 2021-09-13 18:31:50 [gsi-test] [] QuotaTuner... START 2021-09-13 18:31:50 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 47188 KB (expected = 10240 KB), wCtxCnt = 13, objCnt 91, changed reclaimList flush threshold from 50 to 1, changed reclaimSize flush threshold from 10240 KB to 630 KB. 2021-09-13 18:31:51 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: starting... frag 89, data: 33881664, used: 325852938 log:(6739411190 - 7065264128) head:6739411190 2021-09-13 18:31:51 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: completed... frag 89, data: 33881664, used: 317558702, disk: 325852938, relocated: 0, retries: 0, skipped: 61, cleaned:7635014, log:(6747046204 - 7065264128), disk:(6739411190-7065264128), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:12 ms elapsed: 12 ms 2021-09-13 18:31:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 85, data: 12793644, used: 88694364 log:(1271996836 - 1360691200) head:1271996836 2021-09-13 18:31:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 66, data: 20202620, used: 60104686, disk: 62021142, relocated: 395, retries: 0, skipped: 281, cleaned:87830178, log:(1359827014 - 1420791808), disk:(1358770666-1420791808), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:1635 ms elapsed: 1635 ms 2021-09-13 18:31:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoveryCleaner: starting... frag 86, data: 11482324, used: 85069404 log:(4088308132 - 4173377536) head:4088308132 2021-09-13 18:31:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoveryCleaner: completed... frag 69, data: 17973376, used: 58527726, disk: 59919894, relocated: 377, retries: 1, skipped: 272, cleaned:84729506, log:(4173037638 - 4231901184), disk:(4171981290-4231901184), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:1778 ms elapsed: 1778 ms 2021-09-13 18:31:53 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 94, data: 5515800, used: 98836060 log:(51687645604 - 51786481664) head:51687645604 2021-09-13 18:31:53 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 81, data: 14236096, used: 74981358, disk: 76504738, relocated: 501, retries: 1, skipped: 253, cleaned:98361344, log:(51786006948 - 51861458944), disk:(51784954206-51861458944), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:1861 ms elapsed: 1861 ms 2021-09-13 18:31:54 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 99, data: 8256, used: 114954100 log:(59025895564 - 59140849664) head:59025895564 2021-09-13 18:31:54 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 92, data: 8269608, used: 105988078, disk: 106720930, relocated: 693, retries: 1, skipped: 168, cleaned:114217170, log:(59140112734 - 59246833664), disk:(59140112734-59246833664), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:3200 ms elapsed: 3200 ms 2021-09-13 18:31:54 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 67228 KB (expected = 10240 KB), wCtxCnt = 13, objCnt 127, changed reclaimList flush threshold from 50 to 1, changed reclaimSize flush threshold from 10240 KB to 630 KB. 2021-09-13 18:31:55 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 16326 KB (expected = 10240 KB), wCtxCnt = 14, objCnt 30, changed reclaimList flush threshold from 1 to 1, changed reclaimSize flush threshold from 630 KB to 585 KB. 2021-09-13 18:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: starting... frag 85, data: 34529971, used: 240516957 log:(7512993955 - 7753510912) head:7512993955 2021-09-13 18:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: completed... frag 30, data: 46691847, used: 67240464, disk: 71916210, relocated: 214, retries: 0, skipped: 392, cleaned:222653773, log:(7735647728 - 7802888192), disk:(7730971982-7802888192), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:10243 ms elapsed: 10243 ms 2021-09-13 18:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: starting... frag 85, data: 32434928, used: 231206119 log:(8253207321 - 8484413440) head:8253207321 2021-09-13 18:32:01 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: completed... frag 30, data: 43162410, used: 61897558, disk: 67713987, relocated: 237, retries: 0, skipped: 360, cleaned:220307857, log:(8473515178 - 8535412736), disk:(8467698749-8535412736), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:10482 ms elapsed: 10482 ms 2021-09-13 18:32:02 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: starting... frag 94, data: 14647649, used: 254838964 log:(8762877772 - 9017716736) head:8762877772 2021-09-13 18:32:02 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: completed... frag 39, data: 40089106, used: 65822702, disk: 73074506, relocated: 293, retries: 1, skipped: 424, cleaned:253869327, log:(9016747099 - 9083535360), disk:(9010460854-9083535360), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:11190 ms elapsed: 11190 ms 2021-09-13 18:32:07 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 66868 KB (expected = 10240 KB), wCtxCnt = 13, objCnt 298, changed reclaimList flush threshold from 50 to 2, changed reclaimSize flush threshold from 10240 KB to 630 KB. 2021-09-13 18:32:10 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: starting... frag 99, data: 8192, used: 286671252 log:(27305754220 - 27592425472) head:27305754220 2021-09-13 18:32:10 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: completed... frag 86, data: 20585937, used: 152707054, disk: 156992312, relocated: 671, retries: 6, skipped: 147, cleaned:285952049, log:(27591706269 - 27745128448), disk:(27588136136-27745128448), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:19200 ms elapsed: 19200 ms 2021-09-13 18:32:12 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 67115 KB (expected = 10240 KB), wCtxCnt = 14, objCnt 162, changed reclaimList flush threshold from 50 to 1, changed reclaimSize flush threshold from 10240 KB to 585 KB. 2021-09-13T18:32:20.273-07:00 [Info] memstats {"Alloc":99502480, "TotalAlloc":60050454584, "Sys":283252984, "Lookups":0, "Mallocs":4525857,"Frees":4510225, "HeapAlloc":99502480, "HeapSys":263979008, "HeapIdle":161873920, "HeapInuse":102105088,"HeapReleased":102604800, "HeapObjects":15632,"GCSys":8970240, "LastGC":1631583140272188294,"PauseTotalNs":4630338209, "PauseNs":[3728495, 6495993, 6521608, 2544527, 856835, 2920627, 959039, 1504950, 6962105, 2154417, 1677968, 3358740, 1391750, 683890, 1595958, 5453195, 107173936, 5050653, 3680030, 752584, 4127410, 1997492, 2437590, 1840631, 4037247, 1321331, 6096182, 5552938, 3356390, 4788710, 1985433, 4448386, 1600683, 6072194, 4841932, 2034193, 6898536, 2253581, 1804364, 4272507, 3805087, 1796078, 2205323, 1741125, 2472464, 1569208, 1949294, 18677990, 1860753, 10025144, 8228475, 185853812, 1924359, 1422770, 3828842, 12542146, 11327084, 23396239, 19606542, 32248515, 234260100, 3287621, 4247396, 4422678, 5541505, 4113318, 2569441, 957538, 3170023, 2116397, 10800863, 6915947, 5706220, 1422081, 1748347, 1787600, 1477077, 4109645, 1802883, 4572904, 4848341, 1549511, 1819875, 1410759, 1486756, 2333543, 4394600, 2927069, 1880124, 4871831, 3635803, 1727294, 2797354, 2489769, 2890160, 4415985, 1777727, 3217026, 1385278, 1914937, 2028353, 850283, 1304135, 864054, 3739201, 4352874, 4927167, 1514788, 2903629, 4722556, 1143998, 1208582, 4074537, 5391639, 1338794, 651179, 1099127, 3075311, 1730044, 4358246, 2249031, 3482795, 3294494, 630243, 1412039, 1681456, 10658332, 1985054, 2717051, 1324074, 1364051, 35765887, 233864372, 1616998, 3782002, 6300036, 1832950, 1383982, 369872, 1241309, 5453555, 1511657, 2459271, 2222895, 4032574, 1656472, 1735256, 3035094, 3730460, 2476948, 2155372, 1014838, 1641369, 2986053, 6852233, 2999284, 623371, 6512557, 4417593, 2557557, 1105375, 2066604, 1845432, 4044264, 3406941, 1970264, 1381593, 3428586, 1927755, 1729650, 1930057, 1360244, 2280559, 2223716, 1826723, 570564, 4696088, 1183174, 3332606, 3977898, 2004918, 1598652, 5275427, 1080349, 4587291, 960411, 2614705, 1474116, 4245097, 960745, 1204831, 943707, 1214334, 2264777, 3026272, 1435806, 2472765, 3023188, 1455805, 1347965, 3513875, 2537430, 2471109, 1015491, 660604, 1863443, 4307957, 5404227, 2985360, 3514394, 3530531, 6246357, 1567627, 1481541, 2561353, 3194242, 3603076, 2954991, 5056181, 1705358, 5541130, 2805768, 1469040, 1128175, 1925471, 4151837, 3087968, 1241114, 4686693, 3763708, 1652580, 1539795, 3451690, 7981658, 5398548, 2714841, 4585168, 3021438, 3968298, 3958470, 1511817, 2065487, 726041, 7044625, 815181, 1602100, 2125097, 6930398, 1573480, 2220426, 2269438, 4407061, 3609138, 764884, 1043873, 1467397], "NumGC":1042} Allocated: 1761345728, active: 1761476608, metadata: 13282720 (n_thp 0), resident: 1917030400, mapped: 1930072064, retained: 674590720 Mon Sep 13 18:32:22 PDT 2021 RSS = 1756548 2021-09-13 18:32:27 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 26834 KB (expected = 10240 KB), wCtxCnt = 13, objCnt 80, changed reclaimList flush threshold from 50 to 1, changed reclaimSize flush threshold from 10240 KB to 630 KB. 2021-09-13 18:32:29 [plasma] [i] Plasma: mvccPurger: starting... purge_ratio 76.61905 2021-09-13 18:32:30 [plasma] [i] Plasma: mvccPurger: finished... purge_ratio 69.04762 2021-09-13 18:32:50 [ss-workload] instance inse% dele% read% 2021-09-13T18:32:50.413-07:00 [Info] memstats {"Alloc":76582616, "TotalAlloc":95871760800, "Sys":283252984, "Lookups":0, "Mallocs":7480583,"Frees":7461389, "HeapAlloc":76582616, "HeapSys":264011776, "HeapIdle":184901632, "HeapInuse":79110144,"HeapReleased":116195328, "HeapObjects":19194,"GCSys":8908800, "LastGC":1631583170386572636,"PauseTotalNs":7082565757, "PauseNs":[2603690, 1344507, 1634521, 723739, 1438024, 1624454, 1055960, 1393754, 1383813, 1082067, 1467815, 13778668, 2969610, 1762154, 4313306, 1148229, 4262297, 3922757, 4032455, 1305394, 3144219, 1594423, 2023836, 4524301, 5163381, 1491899, 1285575, 4586021, 1679714, 1637498, 2258017, 4134776, 4681945, 1373477, 4967852, 1565848, 1619902, 784757, 2612709, 1217997, 891714, 1307185, 1252701, 3532900, 853823, 832908, 2110551, 1699031, 5060974, 1828382, 1802506, 651002, 7223845, 6115818, 2228630, 11044900, 5205441, 1621864, 3850124, 2859509, 1659192, 4242205, 1651089, 949580, 1274653, 444704, 881458, 904717, 1433551, 1544232, 1501396, 1272062, 3453928, 2446050, 5778324, 1439506, 1846935, 1222160, 1683189, 1135993, 4015671, 1756956, 1157386, 1640600, 975999, 1707405, 5281491, 3123324, 10676627, 1080567, 5405695, 1530253, 1511554, 906327, 3171179, 1852548, 2021949, 4304103, 1007390, 1166746, 1312371, 3074541, 1259244, 1775904, 1573571, 1736906, 4044188, 912016, 3332694, 1117059, 1778361, 1107855, 704886, 3628055, 959120, 976190, 829354, 1165461, 1300257, 1882901, 4286531, 1289195, 2875781, 1059206, 4902881, 1394208, 1043164, 4160902, 1918689, 2210357, 2787416, 1389885, 1328739, 1643536, 1169159, 4919125, 4328862, 1424363, 1247175, 1237045, 1962990, 713469, 5345648, 2236866, 1823264, 2000032, 13678941, 2726331, 5753742, 1630329, 2217286, 4981066, 3620817, 2757777, 4154847, 1153142, 1141531, 1647309, 9249641, 22341667, 2339880, 1205061, 8644977, 5290120, 1501197, 1763187, 1647609, 1109246, 10891594, 1925332, 6476738, 7871625, 4960877, 3651021, 5541117, 1732239, 1919428, 3014946, 1700854, 1588634, 3451003, 1491190, 812766, 505344, 1667746, 1313243, 3702443, 4223571, 5511174, 19553746, 1842545, 1838492, 1028282, 4312835, 1672755, 1348507, 1001075, 2198821, 18583318, 1242230, 1296486, 728957, 1950137, 1362724, 2160662, 3240203, 3759119, 1448338, 970632, 4824058, 1175798, 1275543, 891981, 1413281, 1134509, 1385771, 1173061, 1009453, 1025858, 11434445, 1611870, 2903429, 2261724, 958160, 1076485, 2916956, 964332, 5607624, 4331471, 1436805, 6053212, 1754168, 5350398, 1890248, 1109114, 3394022, 944360, 2217334, 1275431, 861238, 1243121, 1175274, 2288972, 3617629, 1982579, 1111324, 5489987, 576936, 1651503, 3067582, 1544119, 2245199, 2146138, 988338, 923471, 1106526], "NumGC":1684} Allocated: 1721052640, active: 1721163776, metadata: 13282720 (n_thp 0), resident: 1892462592, mapped: 1905504256, retained: 699158528 2021-09-13 18:32:50 [gsi-test] [] error: stats.ItemsCount < 0 2021-09-13 18:32:50 [ss-workload] 0 14% 14% 71% 2021-09-13 18:32:50 [ss-workload] 1 14% 14% 71% 2021-09-13 18:32:50 [ss-workload] 2 14% 14% 71% 2021-09-13 18:32:50 [ss-workload] 3 15% 15% 69% 2021-09-13 18:32:50 [ss-workload] 4 3% 3% 92% 2021-09-13 18:32:50 [ss-workload] 5 3% 3% 92% 2021-09-13 18:32:52 [gsi-test] [] error: stats.ItemsCount < 0 2021-09-13 18:32:50 [ss-workload] 6 3% 3% 92% 2021-09-13 18:32:50 [ss-workload] 7 18% 18% 63% 2021-09-13 18:32:50 [ss-workload] 8 18% 18% 63% 2021-09-13 18:32:50 [ss-workload] 9 18% 18% 62% 2021-09-13 18:32:50 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 18:32:50 [ss-diag] 0 3556 2682 1716 1716 67 25 4391716 208 71552956 13577836 1.000000 2.954885 1.000000 0:0:0 2021-09-13 18:32:50 [ss-diag] 1 4081 3191 1794 1794 101 25 6620348 208 72602556 13577820 1.000000 2.915853 1.000000 0:0:0 2021-09-13 18:32:50 [ss-diag] 2 22204 20600 1333 1347 272 35 17829056 204 118606436 13315436 0.997361 0.586466 0.991347 0:0:0 2021-09-13 18:32:50 [ss-diag] 3 20649 18984 1627 1668 207 26 13568436 208 123257264 13577868 1.000000 0.586466 0.975967 0:0:0 2021-09-13 18:32:50 [ss-diag] 4 67004 66162 1532 1532 556 118 36444688 208 68787292 13577820 1.000000 1.457952 1.000000 0:0:0 2021-09-13 18:32:50 [ss-diag] 5 340397 338809 1616 1616 4196 556 275039408 208 117683844 13577676 1.000000 0.586466 1.000000 0:0:0 2021-09-13 18:32:50 [ss-diag] 6 338734 337110 1606 1606 4527 593 296735796 208 120043348 13578028 1.000000 0.586466 1.000000 0:0:0 2021-09-13 18:32:50 [ss-diag] 7 7827 6915 1564 1564 99 28 6489252 208 73388564 13577628 1.000000 2.580761 1.000000 0:0:0 2021-09-13 18:32:50 [ss-diag] 8 33834 32153 1577 1584 687 83 45031476 208 123911408 13577708 1.000000 0.586466 0.995853 0:0:0 2021-09-13 18:32:50 [ss-diag] 9 43413 41685 1654 1654 738 92 48374424 208 126995540 13577852 1.000000 0.586466 1.000000 0:0:0 2021-09-13 18:33:12 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: starting... frag 94, data: 88741728, used: 1578124807 log:(41257970169 - 42836094976) head:41257970169 2021-09-13 18:33:12 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: completed... frag 88, data: 83488620, used: 715825092, disk: 717942768, relocated: 1970, retries: 56, skipped: 275, cleaned:1577152096, log:(42835122265 - 43551916032), disk:(42833973264-43551916032), activeFragRatio:30, activeMaxFragRatio:80, run:1 duration:81512 ms elapsed: 81512 ms 2021-09-13T18:33:20.663-07:00 [Info] memstats {"Alloc":75936672, "TotalAlloc":134370571096, "Sys":283252984, "Lookups":0, "Mallocs":10636837,"Frees":10618715, "HeapAlloc":75936672, "HeapSys":264175616, "HeapIdle":185679872, "HeapInuse":78495744,"HeapReleased":101261312, "HeapObjects":18122,"GCSys":8912896, "LastGC":1631583200649524513,"PauseTotalNs":8636068315, "PauseNs":[4219875, 4244196, 818423, 3628888, 1712193, 1058084, 2021774, 641080, 2751154, 1907996, 3011587, 1154429, 1483318, 1664602, 1745798, 4669428, 1961460, 1554029, 2412282, 4767520, 3842885, 1099680, 2242347, 4448146, 4570595, 5959304, 1751732, 2480714, 1823590, 1843413, 491018, 2535097, 2126083, 4683492, 603332, 2345180, 5294641, 1003080, 784569, 1116466, 4670941, 1097892, 1332701, 1768855, 991849, 2180871, 1344974, 1800010, 2904423, 1856609, 2161777, 1547332, 1635550, 1186852, 4211503, 3665438, 1400720, 2203000, 1325936, 7250467, 1435478, 1405684, 405611, 2405464, 1198708, 1766975, 1972150, 1335316, 844579, 1134745, 1180019, 1861447, 5088469, 2391063, 950976, 1511492, 1667376, 2378738, 2958578, 1705241, 580173, 1371381, 1826113, 7020492, 1542483, 1981576, 3824622, 1939116, 1802922, 1892719, 564383, 888484, 1082889, 1135967, 1175002, 857858, 961696, 809910, 4199276, 4419790, 628340, 1303658, 1392042, 635714, 1206447, 1630767, 1282587, 2189680, 3389959, 914413, 8706187, 2190234, 2757694, 1929435, 2473740, 2261989, 1379192, 7599490, 1613130, 3571823, 2920834, 1764870, 1163880, 3261554, 2775108, 1037338, 2569685, 6218720, 1005255, 1931787, 2295297, 4114098, 1372315, 1213399, 1593451, 1645502, 1299474, 960749, 5250131, 2849544, 3191058, 1492285, 4690902, 1824196, 1079303, 1471788, 780565, 1203904, 639559, 3572034, 1121991, 2912766, 1938620, 1253168, 3974491, 1227443, 1493079, 978533, 1185294, 1579720, 1097672, 1235866, 3260621, 811158, 2997288, 1172179, 3869886, 1059999, 1429418, 3660214, 796524, 2226317, 1922997, 927222, 991729, 1043156, 1353678, 517954, 1676906, 1067489, 1453409, 998424, 1469874, 2112228, 3617408, 1593652, 1170362, 5216964, 845703, 4186267, 757889, 3197764, 2107416, 2321672, 2139515, 1336297, 3350982, 5170298, 3315778, 1587220, 4293438, 2699823, 1298184, 283975, 1535873, 2985486, 1874118, 4639873, 1716446, 2772218, 3371696, 1074787, 2935432, 4222472, 980631, 1385908, 1205233, 1516397, 1679715, 1023705, 4473901, 1031510, 4061714, 1561391, 2443451, 6766071, 1200290, 1480541, 1636356, 1913552, 2280811, 1536022, 2721593, 586671, 3270243, 3132646, 1352144, 3308619, 4916049, 599219, 3813818, 1330910, 1652974, 2385693, 2260933, 2028438, 1054610, 1309777, 2015583, 4875718, 1052976, 1789975, 1269841, 818527, 4290404, 1485076], "NumGC":2363} Allocated: 1743588352, active: 1743704064, metadata: 13282720 (n_thp 0), resident: 1866117120, mapped: 1879158784, retained: 1195266048 Mon Sep 13 18:33:22 PDT 2021 RSS = 1759472 2021-09-13T18:33:50.878-07:00 [Info] memstats {"Alloc":165527600, "TotalAlloc":173315914800, "Sys":283252984, "Lookups":0, "Mallocs":13791617,"Frees":13772600, "HeapAlloc":165527600, "HeapSys":263880704, "HeapIdle":95903744, "HeapInuse":167976960,"HeapReleased":90873856, "HeapObjects":19017,"GCSys":9011200, "LastGC":1631583230854017594,"PauseTotalNs":10155497763, "PauseNs":[1496920, 1279688, 1430263, 761730, 1925050, 1462561, 4856161, 1630493, 1874199, 3977885, 3352623, 3518230, 1015334, 1357544, 2132342, 1276925, 1499601, 3279144, 3879444, 1016162, 2168303, 1423298, 1467004, 2536792, 1350374, 708475, 2473811, 4484307, 3535422, 5171021, 3113222, 3595907, 1033975, 1505173, 3026999, 1995045, 1594481, 1074791, 3129280, 2867085, 1403465, 1637813, 534345, 1573183, 1531078, 1704980, 2202436, 2630660, 1186164, 1545733, 875994, 4853502, 6738096, 3800866, 1054654, 1819662, 4212185, 2576069, 2934071, 859274, 2599335, 1034257, 672634, 3442675, 1570745, 1636440, 1518197, 677122, 5862037, 5132888, 1446970, 1332844, 1182162, 1304249, 1913472, 3829880, 2934984, 3905261, 4001395, 3769885, 4897886, 1122814, 1428497, 3452577, 733556, 365418, 750026, 1268546, 6440404, 1808858, 4318097, 1145562, 690084, 3777388, 1577619, 1236072, 1111289, 3633178, 1644938, 1443367, 1219553, 1871786, 1759806, 5001431, 5341740, 1152662, 960706, 696875, 942758, 2783675, 3165921, 3897293, 4710349, 2978099, 1864055, 1211576, 2360680, 903901, 1700298, 227690, 1055393, 807564, 1327607, 987797, 960778, 3273366, 1249729, 3275408, 1401441, 5317233, 2119915, 1241549, 423422, 1059643, 1729943, 4314297, 3175176, 1245471, 3819232, 3689772, 916034, 2788098, 1375443, 2875580, 2002637, 536409, 786759, 5548499, 1271252, 1894574, 1048333, 4192021, 1580494, 1262542, 1097108, 1341352, 1332142, 2631164, 1274988, 2030857, 1321517, 1703912, 2513430, 573307, 709903, 1737928, 1947990, 3656567, 3898356, 2142527, 4974068, 1792618, 2649480, 1533089, 1574913, 1325769, 3527548, 1376377, 1567176, 2754023, 2574357, 2463794, 5569429, 2128009, 980758, 1947647, 927484, 4789953, 7027895, 1587419, 2759623, 5145927, 2668424, 1224275, 1376049, 2708895, 699828, 3851361, 2264299, 1582644, 2437878, 1332538, 1395512, 1450431, 664202, 2093674, 1405752, 1180271, 4665293, 1064713, 1650800, 3472650, 1108115, 3071423, 4765489, 5328224, 3208847, 5087224, 1543160, 3167243, 1400832, 513337, 969921, 1242233, 1658191, 1870355, 546029, 2286967, 1570999, 3603184, 1770868, 2218337, 791108, 2894216, 2783794, 1074569, 1497809, 2516691, 1406430, 1236119, 5088347, 926314, 1421180, 3489979, 3853039, 1260068, 635263, 2648437, 732427, 3163081, 4153215, 1508455, 3058124, 756214, 1178666, 3813079], "NumGC":3044} Allocated: 1773777440, active: 1773977600, metadata: 13282720 (n_thp 0), resident: 1892171776, mapped: 1905213440, retained: 1169211392 2021-09-13 18:33:56 [gsi-test] [] error: stats.ItemsCount < 0 2021-09-13 18:33:59 [ss-workload] instance inse% dele% read% 2021-09-13 18:33:59 [ss-workload] 0 14% 14% 70% 2021-09-13 18:33:59 [ss-workload] 1 14% 14% 70% 2021-09-13 18:33:59 [ss-workload] 2 14% 14% 70% 2021-09-13 18:33:59 [ss-workload] 3 14% 14% 71% 2021-09-13 18:33:59 [ss-workload] 4 3% 3% 92% 2021-09-13 18:33:59 [ss-workload] 5 3% 3% 92% 2021-09-13 18:33:59 [ss-workload] 6 3% 3% 92% 2021-09-13 18:33:59 [ss-workload] 7 18% 18% 63% 2021-09-13 18:33:59 [ss-workload] 8 18% 18% 63% 2021-09-13 18:33:59 [ss-workload] 9 18% 18% 62% 2021-09-13 18:33:59 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 18:33:59 [ss-diag] 0 7536 6618 1716 1716 112 29 7341376 208 73781452 13577836 1.000000 2.339045 1.000000 0:0:0 2021-09-13 18:33:59 [ss-diag] 1 7746 6830 1794 1794 84 21 5506032 208 73649540 13577820 1.000000 2.352219 1.000000 0:0:0 2021-09-13 18:33:59 [ss-diag] 2 40607 38907 1347 1347 241 29 15797068 208 125163364 13577836 1.000000 0.640898 1.000000 0:0:1 2021-09-13 18:33:59 [ss-diag] 3 36781 35087 1668 1668 186 23 12191928 208 124765804 13577868 1.000000 0.640898 1.000000 0:0:1 2021-09-13 18:33:59 [ss-diag] 4 128188 127332 1532 1532 595 128 39001060 208 69705492 13577820 1.000000 1.418878 1.000000 0:0:0 2021-09-13 18:33:59 [ss-diag] 5 633453 631877 1616 1616 4506 584 295359288 208 116896980 13577676 1.000000 0.640898 1.000000 0:0:0 2021-09-13 18:33:59 [ss-diag] 6 628432 626846 1606 1606 4683 606 306961284 208 117552668 13578028 1.000000 0.640898 1.000000 0:0:0 2021-09-13 18:33:59 [ss-diag] 7 15546 14546 1564 1564 96 26 6292608 208 79155508 13577628 1.000000 2.229467 1.000000 0:0:0 2021-09-13 18:33:59 [ss-diag] 8 65129 63347 1584 1584 561 68 36772428 208 130533260 13577708 1.000000 0.640898 1.000000 0:0:1 2021-09-13 18:33:59 [ss-diag] 9 75759 73984 1654 1654 410 51 26874680 208 129947168 13577852 1.000000 0.640898 1.000000 0:0:0 2021-09-13T18:34:21.053-07:00 [Info] memstats {"Alloc":97723744, "TotalAlloc":211837629680, "Sys":283252984, "Lookups":0, "Mallocs":16963847,"Frees":16944172, "HeapAlloc":97723744, "HeapSys":263716864, "HeapIdle":163512320, "HeapInuse":100204544,"HeapReleased":103235584, "HeapObjects":19675,"GCSys":8912896, "LastGC":1631583261029788328,"PauseTotalNs":11649590179, "PauseNs":[2018072, 936253, 4372255, 4327165, 1233330, 1560115, 1573034, 1496218, 814934, 5019811, 1325284, 2016864, 1058739, 2412453, 823619, 1857011, 1202327, 3253652, 1005290, 5721009, 1971413, 4427098, 1425465, 5208446, 846794, 3804748, 1737661, 1047502, 1638219, 1409865, 751940, 2013153, 1178215, 3045569, 6303510, 5703618, 1741305, 2741706, 1665818, 1788797, 1173664, 4851497, 2789699, 2387190, 1498625, 3561722, 1101203, 1395493, 2116725, 2402997, 4305477, 872058, 4153168, 1008406, 2637541, 2378595, 2181641, 1733516, 1414449, 1489624, 980834, 4264724, 843290, 923644, 1114802, 3778962, 1346133, 1263510, 542045, 432962, 4644849, 3163542, 2022316, 3957598, 1089409, 5375252, 1020636, 895663, 2512939, 2113174, 784733, 1582055, 991715, 672015, 910910, 3567691, 1312690, 4077220, 1575757, 4163838, 1139671, 3509051, 3067158, 1598691, 3052805, 3728293, 2968809, 5450037, 897777, 3492659, 3313196, 1137756, 2470128, 1485514, 2643198, 1381534, 1637087, 1232244, 1109336, 1637378, 1283036, 1719438, 1269003, 2303575, 3974518, 1444444, 1080407, 1286959, 3263979, 1130301, 3240681, 3777866, 1661805, 1799303, 2982321, 1253688, 2319650, 5996928, 2902731, 1082871, 2089489, 1303035, 1551291, 1985585, 1232263, 1654007, 3995211, 1982649, 6618567, 1067922, 4224135, 2402094, 1459676, 1717954, 1375740, 839288, 872813, 3583526, 2422839, 645344, 4653472, 1225506, 1249779, 1541743, 1005242, 2306428, 2048876, 1484798, 2094849, 1253544, 1495883, 2187625, 1301877, 3045955, 1894168, 968030, 1704159, 5620358, 4595971, 3333262, 2274321, 4733918, 794213, 773090, 2760766, 2953095, 1272695, 1015324, 1024860, 1976926, 1269877, 1550113, 1276758, 3148751, 1260330, 1195853, 2919344, 2218258, 1292456, 1289192, 1034883, 4929111, 1579359, 1448537, 2124609, 3538782, 929922, 342625, 1379462, 1351669, 3261797, 1288877, 4454054, 2891113, 3971756, 3929701, 2954232, 700685, 708791, 3411929, 1238420, 1688874, 1116602, 2501036, 2557034, 1805840, 5307676, 1790074, 588034, 3553254, 3245858, 746486, 1227595, 1881980, 6597624, 3411113, 2236561, 2148247, 652631, 1530335, 1160148, 3761089, 4065244, 3604191, 221905, 1070827, 4628926, 1387776, 3458394, 2057394, 1416944, 1437463, 4497876, 2779430, 1757359, 1043320, 1544429, 1639680, 2800071, 1053988, 1129403, 1807251, 905575, 1541772, 1816083, 1374729], "NumGC":3713} Allocated: 1757064080, active: 1758048256, metadata: 13282720 (n_thp 0), resident: 1863921664, mapped: 1877749760, retained: 1196675072 Mon Sep 13 18:34:22 PDT 2021 RSS = 1775488 2021-09-13T18:34:51.277-07:00 [Info] memstats {"Alloc":92223784, "TotalAlloc":250315631200, "Sys":283252984, "Lookups":0, "Mallocs":20109674,"Frees":20092727, "HeapAlloc":92223784, "HeapSys":263323648, "HeapIdle":168452096, "HeapInuse":94871552,"HeapReleased":109862912, "HeapObjects":16947,"GCSys":9011200, "LastGC":1631583291256531203,"PauseTotalNs":13189875121, "PauseNs":[1194525, 1588517, 1690208, 2902916, 1456823, 1289790, 1774146, 2187780, 1248565, 5351811, 1932335, 2739502, 920371, 4168755, 1489057, 738267, 1098618, 1855702, 1680125, 4399180, 3502109, 995797, 5850039, 1678297, 1021011, 1184613, 1722034, 6864370, 1147400, 1176120, 1098219, 1457270, 2256714, 1237866, 1588877, 1623527, 4473031, 837932, 3293204, 1322789, 1476364, 1276815, 3176172, 4021564, 1315334, 4533134, 1515655, 1679035, 3954399, 710911, 3887070, 3680094, 3333362, 1861046, 1322103, 3001643, 1702112, 1092429, 4097074, 2409295, 1172057, 1619575, 2404613, 4041756, 4362092, 4155711, 1285709, 1551636, 1298996, 2449534, 3054826, 5659828, 525662, 5929083, 2667052, 2073377, 1198407, 3014968, 856291, 1736322, 3853960, 361462, 3360999, 1713604, 1520397, 2176732, 1970470, 1122198, 2969235, 1259846, 1305489, 4603420, 3433922, 992586, 834780, 3097580, 3942592, 2466483, 2097418, 6254155, 979479, 1922133, 1962655, 1612647, 3709062, 1680712, 4047973, 859015, 2037857, 1307787, 959101, 1599494, 1410178, 1258518, 1016743, 1418240, 1044536, 1027675, 1164415, 1776084, 3196075, 2798618, 2536717, 507422, 2821073, 964359, 1705024, 1470667, 3279568, 1111161, 2515851, 6004051, 2835926, 938579, 1977327, 3501004, 1857239, 1753888, 5334667, 1095219, 739006, 1047768, 780671, 1919291, 3360535, 847370, 746501, 1275830, 460359, 1143296, 4970472, 3653418, 1574690, 2367201, 1161577, 892970, 2697836, 1567706, 1096631, 1268899, 3151069, 5491344, 1381777, 1060212, 968384, 1105169, 1389521, 4105849, 1366903, 1189966, 1059053, 2085749, 1953451, 2796102, 1948542, 6214554, 3167854, 409522, 4719253, 871180, 2896948, 4110201, 5595610, 4536402, 1385911, 1185604, 1214570, 1681434, 1338301, 685889, 3551585, 1633418, 6481249, 1443925, 2173340, 2825143, 1198861, 5369177, 2898765, 3423275, 4576167, 4732103, 1306101, 6793388, 1043179, 2046975, 3053196, 934554, 5664358, 1512960, 1224250, 1066875, 1577967, 3458918, 1648741, 3455363, 2942604, 1904714, 1424825, 1771016, 3090835, 1352812, 2458526, 2211887, 1738596, 5410843, 2008780, 1209246, 8032740, 2126313, 1689764, 2609792, 1596843, 1645855, 1802923, 1619491, 1064236, 4446908, 1569425, 1281381, 2817946, 2582600, 7421650, 4592325, 5535090, 622206, 4000575, 1285129, 1898388, 1115898, 3075421, 1314907, 6644471, 1219023, 4781743, 1843896], "NumGC":4385} Allocated: 1585566040, active: 1586700288, metadata: 13282720 (n_thp 0), resident: 1792790528, mapped: 1805832192, retained: 1268592640 2021-09-13 18:35:00 [gsi-test] [] QuotaTuner: 1516282026 -> 1364653823 2021-09-13 18:35:06 [gsi-test] [] error: stats.ItemsCount < 0 2021-09-13 18:35:08 [ss-workload] instance inse% dele% read% 2021-09-13 18:35:08 [ss-workload] 0 14% 14% 70% 2021-09-13 18:35:08 [ss-workload] 1 14% 14% 70% 2021-09-13 18:35:08 [ss-workload] 2 14% 14% 70% 2021-09-13 18:35:08 [ss-workload] 3 14% 14% 70% 2021-09-13 18:35:08 [ss-workload] 4 3% 3% 92% 2021-09-13 18:35:08 [ss-workload] 5 3% 3% 92% 2021-09-13 18:35:08 [ss-workload] 6 3% 3% 92% 2021-09-13 18:35:08 [ss-workload] 7 18% 18% 63% 2021-09-13 18:35:08 [ss-workload] 8 18% 18% 63% 2021-09-13 18:35:08 [ss-workload] 9 18% 18% 62% 2021-09-13 18:35:08 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 18:35:08 [ss-diag] 0 11301 10381 1716 1716 106 29 6948088 208 73914196 13577836 1.000000 2.434295 1.000000 0:0:0 2021-09-13 18:35:08 [ss-diag] 1 11237 10297 1794 1794 77 23 5047196 208 75224452 13577820 1.000000 2.509358 1.000000 0:0:0 2021-09-13 18:35:08 [ss-diag] 2 58088 56378 1347 1347 327 42 21434196 208 125818732 13577836 1.000000 0.605701 1.000000 0:0:0 2021-09-13 18:35:08 [ss-diag] 3 53647 52003 1668 1668 294 40 19271112 208 121357028 13577868 1.000000 0.605701 1.000000 0:0:0 2021-09-13 18:35:08 [ss-diag] 4 184982 184158 1532 1532 1038 211 68038824 208 67608724 13577820 1.000000 1.485021 1.000000 0:0:0 2021-09-13 18:35:08 [ss-diag] 5 925424 923794 1616 1616 5044 650 330624112 208 120436812 13577676 1.000000 0.605701 1.000000 0:0:0 2021-09-13 18:35:08 [ss-diag] 6 919268 917684 1606 1606 4214 553 276219272 208 117421172 13578028 1.000000 0.605701 1.000000 0:0:0 2021-09-13 18:35:08 [ss-diag] 7 21705 20747 1564 1564 144 36 9438912 208 76402124 13577628 1.000000 2.471221 1.000000 0:0:0 2021-09-13 18:35:08 [ss-diag] 8 94030 92332 1584 1584 308 39 20188784 208 124895452 13577708 1.000000 0.605701 1.000000 0:0:0 2021-09-13 18:35:08 [ss-diag] 9 110755 109013 1654 1654 501 64 32839548 208 127783052 13577852 1.000000 0.605701 1.000000 0:0:0 2021-09-13T18:35:21.454-07:00 [Info] memstats {"Alloc":122003048, "TotalAlloc":288523515080, "Sys":283252984, "Lookups":0, "Mallocs":23231079,"Frees":23214395, "HeapAlloc":122003048, "HeapSys":263225344, "HeapIdle":138567680, "HeapInuse":124657664,"HeapReleased":113827840, "HeapObjects":16684,"GCSys":9043968, "LastGC":1631583321438325203,"PauseTotalNs":14735387567, "PauseNs":[766285, 3030422, 1275650, 1223761, 1256612, 1053939, 956485, 2669749, 1166508, 712444, 1516477, 946828, 3114114, 878440, 994901, 1321504, 1267500, 1336554, 2184471, 811879, 538143, 1652780, 1032087, 3440087, 1501779, 1523896, 1657150, 1304140, 1771996, 1219984, 1532647, 1886584, 1455419, 2539333, 1985624, 3177019, 1258312, 1379852, 1278922, 1588786, 4010736, 1566492, 1972161, 1255767, 1020874, 6445869, 2847492, 1842168, 1504972, 1957226, 507619, 957168, 3352851, 4200457, 508503, 2198727, 6135104, 5316544, 6370151, 6994059, 1619511, 5118314, 1680454, 3962672, 2506569, 1491598, 2280402, 485020, 1404889, 1512048, 898402, 1010419, 2143588, 1855696, 1920441, 1843811, 1202697, 1546415, 6272458, 4119034, 3327278, 1229067, 2244580, 3623674, 797672, 882750, 1840287, 3229417, 344495, 1793926, 1098439, 1662955, 1938519, 1194104, 916819, 1769478, 3292655, 5886342, 3890800, 2479500, 797897, 1622375, 2928958, 1307846, 7769334, 1253847, 1828758, 3202815, 2057208, 5113088, 1014667, 908294, 797842, 1367987, 1026283, 1523484, 1798883, 833182, 1337513, 1575205, 2271949, 1524528, 2380490, 2812687, 1580325, 1176954, 996871, 2255018, 885420, 3595277, 2640755, 1141133, 1274142, 2880774, 1380474, 1259205, 1944035, 3365732, 1441897, 1670924, 1818291, 2247622, 2656397, 1716860, 1582456, 863514, 4570575, 963279, 1780860, 794908, 1714963, 1447679, 1656547, 2082140, 1772010, 1788122, 3681528, 1836720, 3789159, 1368796, 3829716, 1133372, 925395, 1817039, 1198885, 1914091, 1372382, 3074963, 5099958, 6459743, 5150890, 2485996, 2750424, 1150730, 1165442, 6778105, 1776081, 1779408, 1235147, 3970114, 1068878, 1373872, 1204147, 3449017, 2452056, 1423451, 1769650, 1761188, 1447992, 2218578, 1223390, 3981509, 3372523, 3238221, 2472720, 5881385, 656584, 4435587, 970420, 1700730, 1305920, 4996029, 5287075, 1865368, 2052675, 1319382, 1551085, 1486037, 989115, 2821292, 1158525, 1380405, 1411260, 2724812, 1262122, 3322177, 1162653, 2903011, 3615455, 2794220, 2962686, 840593, 747899, 3457874, 1524203, 1913152, 845785, 1165547, 1565729, 719294, 2632036, 1150142, 1365147, 740256, 2570188, 1847409, 3371912, 4223352, 983369, 1443295, 4957230, 1641438, 2364308, 2804214, 2322704, 2132551, 1758508, 2502320, 3550754, 3958941, 6690823, 1720879, 877657, 1925391, 1568429, 1581439], "NumGC":5060} Allocated: 1591405048, active: 1592496128, metadata: 13311424 (n_thp 0), resident: 1723842560, mapped: 1736884224, retained: 1337540608 Mon Sep 13 18:35:22 PDT 2021 RSS = 1627660 2021-09-13T18:35:51.633-07:00 [Info] memstats {"Alloc":60646832, "TotalAlloc":327594914440, "Sys":283252984, "Lookups":0, "Mallocs":26352679,"Frees":26335949, "HeapAlloc":60646832, "HeapSys":263487488, "HeapIdle":200130560, "HeapInuse":63356928,"HeapReleased":110903296, "HeapObjects":16730,"GCSys":8912896, "LastGC":1631583351624813021,"PauseTotalNs":16169030836, "PauseNs":[1606763, 843503, 1604901, 1509560, 1014886, 2992437, 811960, 929401, 2005518, 1891472, 4468689, 677328, 2860353, 1528140, 1406556, 2313838, 2272269, 3277213, 1987908, 978131, 1178372, 3324359, 1440470, 1825165, 1568888, 2175263, 2473850, 4829087, 754673, 2542976, 3390892, 1056277, 1235554, 1260087, 4337814, 1001738, 2878863, 3300646, 1536715, 1354205, 1883592, 1335642, 1285773, 1299581, 1415607, 4223646, 1078782, 1128041, 3460697, 1980023, 1348358, 1190049, 1187934, 901860, 973767, 2852975, 1507655, 1683466, 1593307, 1654700, 1470247, 2128645, 1307688, 803102, 1899291, 2045141, 3514575, 2502848, 1624910, 1656091, 2747408, 1488857, 1352081, 2767895, 1176805, 1538638, 1926047, 5348085, 879953, 1389442, 3102355, 1475254, 1397187, 1368614, 461797, 1228184, 1360334, 1746943, 993598, 2020582, 3548016, 1965467, 1675920, 1196957, 723264, 1766971, 2352921, 3382620, 3226070, 1537022, 1385399, 1093556, 1591265, 3372154, 1456258, 1414793, 1183079, 1760535, 3224496, 1505117, 1200518, 1821149, 1485091, 1196420, 2075184, 2135006, 1127971, 2257120, 913809, 1921749, 6526434, 1521282, 1969436, 2506681, 1166351, 1460855, 773026, 690792, 1294311, 3480304, 1452645, 1746822, 1528802, 4092105, 2663892, 2559539, 2864665, 1067855, 4303879, 1106240, 1472944, 4665553, 4769714, 1028725, 1990855, 1521556, 2120156, 4265925, 752383, 840584, 892251, 2066586, 4018448, 2859606, 1467447, 1842987, 1126365, 2737326, 2275105, 1301537, 3971641, 1786511, 5210620, 1324320, 3122677, 1480882, 2377418, 1028830, 1396594, 1947332, 1013758, 896370, 1389912, 1361283, 5291196, 3229191, 3761405, 2001872, 2006471, 2956708, 3659972, 2087860, 1232520, 1302040, 627439, 1628892, 2452278, 1608940, 1535878, 1440488, 1273509, 2596500, 4588997, 5083908, 1378126, 1573940, 2650793, 1443079, 2667351, 1453458, 1377785, 1553413, 982207, 2084337, 3788522, 1043159, 1496020, 1548350, 738451, 1393600, 1609120, 497891, 1393945, 2929464, 1008834, 1332484, 999524, 2426274, 1620358, 1419913, 1328966, 3547294, 1375313, 6098075, 1099198, 1716276, 1886594, 1540368, 1719145, 4626400, 1238145, 1908064, 2136532, 1314428, 771887, 4978774, 1723642, 1713097, 1469313, 851435, 942448, 1626685, 1928806, 1540495, 1642473, 1611625, 1047659, 2636516, 4677759, 1131347, 1043197, 2924321, 1479733, 2251866, 1553155, 1475862], "NumGC":5743} Allocated: 1619203888, active: 1620295680, metadata: 13311424 (n_thp 0), resident: 1747501056, mapped: 1760542720, retained: 1313882112 2021-09-13 18:36:18 [ss-workload] instance inse% dele% read% 2021-09-13 18:36:18 [ss-workload] 0 14% 14% 70% 2021-09-13 18:36:18 [ss-workload] 1 14% 14% 70% 2021-09-13 18:36:18 [ss-workload] 2 14% 14% 70% 2021-09-13 18:36:18 [ss-workload] 3 14% 14% 70% 2021-09-13 18:36:18 [ss-workload] 4 3% 3% 92% 2021-09-13 18:36:18 [ss-workload] 5 3% 3% 92% 2021-09-13 18:36:18 [ss-workload] 6 3% 3% 92% 2021-09-13T18:36:21.831-07:00 [Info] memstats {"Alloc":122640160, "TotalAlloc":364841384768, "Sys":283252984, "Lookups":0, "Mallocs":29321410,"Frees":29303509, "HeapAlloc":122640160, "HeapSys":263159808, "HeapIdle":137805824, "HeapInuse":125353984,"HeapReleased":84230144, "HeapObjects":17901,"GCSys":9011200, "LastGC":1631583381818803774,"PauseTotalNs":17698716093, "PauseNs":[1091598, 2435856, 4987491, 1137468, 1589863, 2822053, 1144005, 816594, 1579730, 1187303, 1531510, 2185377, 3224430, 1476738, 839082, 1502524, 1216263, 1997512, 1663875, 927320, 1025698, 1336988, 2097958, 1103083, 1326093, 4132622, 1561500, 5766927, 1876981, 1972218, 1603820, 1753767, 1860808, 1945291, 1555449, 3448929, 3473111, 3102434, 1771542, 1532371, 1695575, 1161763, 2656258, 665901, 2956914, 705702, 1555620, 3307483, 2253477, 1602446, 1022925, 4529180, 1695351, 1379227, 2018913, 4396664, 1683037, 1409017, 1262700, 1362447, 428244, 2179457, 391620, 3602144, 1801481, 1080713, 6044756, 1334932, 1542767, 1467457, 1655516, 3051852, 1237611, 1287769, 1840770, 8022785, 1672193, 1380640, 521424, 1515345, 2669865, 2527524, 2950313, 1874058, 2247001, 3383971, 2816990, 2304191, 4950970, 1353376, 1473955, 5658144, 2672712, 3473657, 2018269, 5314089, 4568197, 904912, 826100, 1280901, 2343933, 1777245, 1049691, 1073156, 1505035, 1077093, 1277979, 4475524, 3849544, 3761501, 6058738, 3548754, 5350217, 4947238, 842183, 1715188, 4149059, 1722310, 1327836, 1440901, 1828121, 971570, 2800555, 3087064, 1716827, 869588, 1701555, 647634, 3926740, 1447315, 984277, 646922, 1410250, 3771718, 1501563, 852840, 2262893, 1084111, 1815724, 3527218, 2371665, 1705667, 2664101, 1856475, 2262221, 1756326, 4138751, 4384400, 456636, 1030047, 2045777, 2045616, 4377398, 1251615, 3365013, 1294702, 4302922, 2109562, 2068598, 1707517, 1645650, 4333681, 924550, 4405293, 2042173, 1105954, 2497168, 2594915, 1510169, 1496051, 1093861, 1215564, 1819652, 1345155, 1019719, 1486862, 3380558, 6571719, 1859221, 1627996, 3709100, 1644951, 2899413, 3178338, 1825392, 6158409, 5430245, 3277037, 1254363, 2635049, 593957, 1195944, 2221550, 1865192, 3747153, 3317069, 2256640, 2665907, 922945, 2216597, 927115, 1776761, 4323701, 2143494, 5934251, 1406155, 1200070, 4655973, 2265399, 1169185, 1175518, 1746326, 4550915, 1383955, 1929107, 3105308, 751483, 4346041, 2523058, 1126784, 1587971, 4129721, 876001, 1903078, 1998597, 1314857, 1395903, 951938, 3008381, 7605287, 1804951, 4736341, 782405, 480004, 1116653, 1080545, 2950197, 1125427, 1796159, 1381420, 1318192, 1585294, 1820285, 3981653, 1360821, 1436881, 2744742, 1122496, 3952948, 5272418, 2874373, 1829513, 630837, 7186032, 1453767, 4652790], "NumGC":6403} Allocated: 1659826376, active: 1661198336, metadata: 13311424 (n_thp 0), resident: 1811800064, mapped: 1824841728, retained: 1249583104 2021-09-13 18:36:18 [ss-workload] 7 18% 18% 63% Mon Sep 13 18:36:22 PDT 2021 RSS = 1685496 2021-09-13 18:36:18 [ss-workload] 8 18% 18% 62% 2021-09-13 18:36:18 [ss-workload] 9 18% 18% 62% 2021-09-13 18:36:18 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 18:36:18 [ss-diag] 0 15291 14367 1716 1716 91 24 5964868 208 74175140 13577836 1.000000 2.366487 1.000000 0:0:0 2021-09-13 18:36:18 [ss-diag] 1 15364 14474 1794 1794 82 21 5374936 208 71945628 13577820 1.000000 2.348665 1.000000 0:0:0 2021-09-13 18:36:18 [ss-diag] 2 76157 74475 1347 1347 241 31 15797068 208 123850588 13577836 1.000000 0.602280 1.000000 0:0:0 2021-09-13 18:36:18 [ss-diag] 3 71359 69663 1668 1668 350 42 22941800 208 124765620 13577868 1.000000 0.602280 1.000000 0:0:0 2021-09-13 18:36:18 [ss-diag] 4 246788 245940 1532 1532 778 161 50996344 208 69180820 13577820 1.000000 1.322411 1.000000 0:0:0 2021-09-13 18:36:18 [ss-diag] 5 1223003 1221387 1616 1616 4978 646 326297944 208 119518580 13577676 1.000000 0.602280 1.000000 0:0:0 2021-09-13 18:36:18 [ss-diag] 6 1196399 1194811 1606 1606 5180 672 339538640 208 117683332 13578028 1.000000 0.602280 1.000000 0:0:0 2021-09-13 18:36:18 [ss-diag] 7 28196 27210 1564 1564 94 26 6161512 208 78238908 13577628 1.000000 2.193179 1.000000 0:0:0 2021-09-13 18:36:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 202, "num_active_sctxs": 46, "num_free_sctxs": 155, "num_trim_sctxs": 174, "num_wctxs": 247, "num_free_wctxs": 225, "num_active_wctxs": 22, "num_trim_wctxs": 106, "num_readers": 102, "num_writers": 103, "cpu": 9.79167, "total_core": 16, "threads": 48, "assigned_quota": 1364653823, "current_quota": 1364653823, "memory_resident": 0, "memory_jemalloc": 1728262144, "memory_jemalloc_allocated": 1581373872, "memory_in_use": 1728917504, "memory_stats_size_total": 1294686568, "memory_stats_size_global": 1171079816, "memory_stats_size_page": 1035709248, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 209336, "reclaim_pending": 13180452, "buf_memused": 110018796, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 13, "num_active_smr": 12, "num_pending_smr": 1, "num_pages": 2080, "items_count": 24, "total_records": 13726, "total_ops": 114763, "compacts": 410296, "purges": 128, "lss_data_size": 297413520, "lss_used_space": 1231551280, "lss_fragmentation": 75, "lss_num_reads": 265900, "lss_read_bs": 40891458677, "lss_blk_read_bs": 40721211392, "rlss_num_reads": 5518, "lss_rdr_reads_bs": 1937326572, "lss_blk_rdr_reads_bs": 670752768, "bytes_written": 36209704960, "bytes_incoming": 7522485124, "write_amp_avg": 1.51, "num_swapper_ctx": 3, "avg_sweep_interval": 300 s, "avg_run_interval": 100 ms, "avg_clock_interval": 0 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 13726, "resident_ratio": 100.00000, "cleaner_num_reads": 260344, "cleaner_read_bs": 38232665882, "cleaner_blk_read_bs": 39333838848, "recovery_num_lssctx": 5, "recovery_num_sctxs": 46, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 26, "recovery_num_trim_sctxs": 26, "recovery_buf_memused": 5242880, "recovery_data_size": 95874612, "recovery_used_space": 362752308, "recovery_lss_blk_read_bs": 30370955264, "recovery_lss_read_bs": 29588109312, "recovery_lss__num_reads": 224396, "recovery_bytes_written": 28633116672, "recovery_lss_frag": 73, "recovery_cleaner_num_reads": 224396, "recovery_cleaner_read_bs": 28900562389, "recovery_cleaner_blk_read_bs": 29684846592, "mvcc_purge_ratio": 571.91667 }2021-09-13 18:36:18 [ss-diag] 8 126214 124456 1584 1584 682 86 44703736 208 128828876 13577708 1.000000 0.602280 1.000000 0:0:0 2021-09-13 18:36:18 [ss-diag] 9 139368 137624 1654 1654 441 53 28906668 208 127916020 13577852 1.000000 0.602280 1.000000 0:0:0 2021-09-13 18:36:51 [gsi-test] [] Stop and verify... 2021-09-13 18:36:51 [gsi-test] [] { "memory_quota": 1364653823, "count": 4670, "compacts": 3118, "purges": 0, "splits": 0, "merges": 0, "inserts": 4670, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 74830188, "memory_size_index": 13577836, "allocated": 1337110856, "freed": 1262280668, "reclaimed": 1262280668, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 1, "allocated_index": 13577836, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 934, "num_rec_allocs": 16487, "num_rec_frees": 15553, "num_rec_swapout": 1716, "num_rec_swapin": 1716, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 306109160, "lss_data_size": 32553615, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 47179152, "est_recovery_size": 28597288, "lss_num_reads": 478, "lss_read_bs": 174978853, "lss_blk_read_bs": 77451264, "lss_rdr_reads_bs": 149526290, "lss_blk_rdr_reads_bs": 51814400, "cache_hits": 19284, "cache_misses": 138, "cache_hit_ratio": 1.00000, "rlss_num_reads": 408, "rcache_hits": 14614, "rcache_misses": 138, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 0.00000, "currSn": 362652, "gcSn": 362649, "gcSnIntervals": "[0 152276 181788 211314 240838 362650]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 4, "num_free_wctxs": 17, "num_trim_wctxs": 21, "num_readers": 9, "num_writers": 6, "page_bytes": 7931308, "page_cnt": 27, "page_itemcnt": 121, "avg_item_size": 65548, "avg_page_size": 293752, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 88145864, "page_bytes_marshalled":9630551420, "page_bytes_compressed":5243253035, "compression_ratio": 1.83675, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 34128, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 5302224, "lss_data_size": 51117067, "lss_used_space": 74728388, "lss_disk_size": 171876352, "lss_fragmentation": 31, "lss_num_reads": 56406, "lss_read_bs": 7854599838, "lss_blk_read_bs": 8041451520, "bytes_written": 7365136384, "bytes_incoming": 306109160, "write_amp": 2.63, "write_amp_avg": 2.42, "lss_gc_num_reads": 55998, "lss_gc_reads_bs": 7616811066, "lss_blk_gc_reads_bs": 7900942336, "lss_gc_status": "frag 30, data: 32553615, used: 46945430, disk: 48738160, relocated: 425318, retries: 24, skipped: 111904, cleaned:709645578, log:(8445293306 - 8492920832), disk:(8444182672-8492920832), activeFragRatio:30, activeMaxFragRatio:80, run:248 duration:72806 ms elapsed: 289193 ms", "lss_head_offset": 8445293306, "lss_tail_offset": 8492920832, "num_sctxs": 28, "num_free_sctxs": 20, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:36:51 [gsi-test] [] { "memory_quota": 1364653823, "count": 4794, "compacts": 3203, "purges": 0, "splits": 0, "merges": 0, "inserts": 4794, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 4, "persist_conflicts": 0, "memory_size": 74436628, "memory_size_index": 13577820, "allocated": 1354948680, "freed": 1280512052, "reclaimed": 1277691720, "reclaim_pending": 2820332, "reclaim_list_size": 2820332, "reclaim_list_count": 7, "reclaim_threshold": 1, "allocated_index": 13577820, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 928, "num_rec_allocs": 16742, "num_rec_frees": 15814, "num_rec_swapout": 1794, "num_rec_swapin": 1794, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 314237112, "lss_data_size": 32516511, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 47125378, "est_recovery_size": 28269410, "lss_num_reads": 524, "lss_read_bs": 198288112, "lss_blk_read_bs": 81231872, "lss_rdr_reads_bs": 179536760, "lss_blk_rdr_reads_bs": 62328832, "cache_hits": 20162, "cache_misses": 163, "cache_hit_ratio": 1.00000, "rlss_num_reads": 475, "rcache_hits": 15368, "rcache_misses": 163, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 0.00000, "currSn": 362650, "gcSn": 362648, "gcSnIntervals": "[0 153392 182904 212424 241944 362649]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 6, "num_free_wctxs": 24, "num_trim_wctxs": 1, "num_readers": 12, "num_writers": 10, "page_bytes": 7800212, "page_cnt": 31, "page_itemcnt": 119, "avg_item_size": 65548, "avg_page_size": 251619, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 85916240, "page_bytes_marshalled":9446297218, "page_bytes_compressed":5143317756, "compression_ratio": 1.83662, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 34008, "num_total_smr": 1, "num_active_smr": 3, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 5655840, "lss_data_size": 51211095, "lss_used_space": 74252392, "lss_disk_size": 101609472, "lss_fragmentation": 31, "lss_num_reads": 56412, "lss_read_bs": 7871955450, "lss_blk_read_bs": 8030572544, "bytes_written": 7375159296, "bytes_incoming": 314237112, "write_amp": 2.52, "write_amp_avg": 2.41, "lss_gc_num_reads": 55937, "lss_gc_reads_bs": 7607762992, "lss_blk_gc_reads_bs": 7883173888, "lss_gc_status": "frag 30, data: 32516511, used: 46731718, disk: 48201353, relocated: 440558, retries: 116, skipped: 102934, cleaned:721987495, log:(9195502673 - 9242484736), disk:(9194283383-9242484736), activeFragRatio:30, activeMaxFragRatio:80, run:247 duration:76148 ms elapsed: 289416 ms", "lss_head_offset": 9195502673, "lss_tail_offset": 9242484736, "num_sctxs": 29, "num_free_sctxs": 19, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13T18:36:51.959-07:00 [Info] memstats {"Alloc":77195352, "TotalAlloc":401882454928, "Sys":283252984, "Lookups":0, "Mallocs":32282154,"Frees":32266505, "HeapAlloc":77195352, "HeapSys":263356416, "HeapIdle":183402496, "HeapInuse":79953920,"HeapReleased":115539968, "HeapObjects":15649,"GCSys":8978432, "LastGC":1631583411949723678,"PauseTotalNs":19143945283, "PauseNs":[2624179, 608770, 4085073, 1011818, 2379520, 2587896, 1252666, 6245190, 907853, 1381971, 3286357, 3532459, 2502117, 1867501, 1256592, 1745276, 2437039, 3372727, 4387708, 2279673, 2712179, 2931240, 1857931, 2071647, 1267220, 1186743, 3723085, 1644661, 1520784, 3575703, 1010802, 2117925, 1363760, 1287499, 2553750, 674095, 6564651, 5635684, 4150871, 1613817, 1089757, 1960287, 1684833, 1948780, 2336561, 3244886, 2148521, 777840, 2668307, 2652152, 2232854, 6698771, 897670, 596871, 2377690, 2549491, 2547713, 4327301, 1347552, 1453389, 1770940, 1039927, 1911003, 1240743, 620908, 1805853, 2100464, 1516262, 1877389, 1500543, 5124602, 1774433, 2435817, 2044808, 4980685, 4019260, 1279204, 1241434, 1098241, 1033310, 986684, 1336877, 1141349, 1465717, 963409, 1680763, 1067603, 7126877, 1852242, 1206213, 1273614, 1566971, 1625221, 1560176, 2937723, 1360995, 4103541, 1688274, 2602684, 5459961, 1371938, 1830922, 1089130, 1347774, 1758040, 1063124, 1821816, 2411268, 1454710, 2110653, 2821103, 3003065, 1915245, 4438932, 3632833, 4273087, 1607835, 1496121, 2019331, 1283429, 2005811, 2037138, 3175318, 1748514, 782092, 2636731, 768826, 1337031, 932374, 2156974, 3516777, 871518, 5526426, 1230431, 2550468, 2514640, 5838763, 1220943, 994388, 2865079, 1421408, 3720404, 3130237, 1396020, 3291260, 1217928, 4568602, 2662887, 2745430, 1732244, 1883207, 1839378, 1788923, 1269905, 1015710, 1481681, 4763317, 2412153, 5838161, 1828347, 7245787, 1380434, 2374082, 2586749, 2080610, 2145629, 1549007, 1666568, 1121000, 1029537, 1557926, 2755542, 1633602, 3005630, 1965472, 1006855, 1550420, 5293702, 2335049, 2008529, 1898808, 1437974, 1514070, 5691254, 1603138, 1089416, 1297385, 1526862, 1636574, 1421511, 2741556, 2457109, 758135, 949716, 2213063, 1588309, 1455840, 4353048, 4332115, 2970131, 1948748, 1147041, 3042322, 514218, 1620190, 3566348, 4489580, 425483, 1793959, 898521, 1408425, 1401806, 1135687, 3143946, 1240812, 4772119, 6193810, 4098493, 4028992, 1998680, 453075, 1049115, 1020211, 1904438, 1359855, 2042168, 2120625, 1883175, 1500048, 1466683, 2492568, 2226582, 2735353, 2264364, 748943, 948444, 3263554, 1637728, 2647666, 1275337, 1146195, 3898076, 850477, 1528102, 1501113, 4399362, 1549948, 1412577, 1154856, 2703534, 765054, 1933919, 1836619, 486476, 1869070, 2127068], "NumGC":7061} Allocated: 1584182976, active: 1585618944, metadata: 13368832 (n_thp 0), resident: 1665835008, mapped: 1678876672, retained: 1395548160 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: starting... frag 89, data: 35127400, used: 320323502 log:(6747046204 - 7068028928) head:6747046204 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: completed... frag 94, data: 14736356, used: 249310822, disk: 252931604, relocated: 0, retries: 0, skipped: 1259985, cleaned:1056317288, log:(7803363492 - 8053067776), disk:(7800136172-8053067776), activeFragRatio:30, activeMaxFragRatio:80, run:301 duration:3415 ms elapsed: 300012 ms 2021-09-13 18:36:52 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper stopped 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:36:52 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : All daemons stopped 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper stopped 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:36:52 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : All daemons stopped 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper stopped 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:36:52 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : All daemons stopped 2021-09-13 18:36:52 [gsi-test] [0] Stop and verify... 2021-09-13 18:36:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:36:52 [gsi-test] [] { "memory_quota": 1364653823, "count": 4427, "compacts": 9875, "purges": 128, "splits": 30, "merges": 0, "inserts": 4427, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 1, "delete_conflicts": 0, "swapin_conflicts": 2, "persist_conflicts": 0, "memory_size": 121620596, "memory_size_index": 13577836, "allocated": 6124941184, "freed": 6003320588, "reclaimed": 6002796028, "reclaim_pending": 524560, "reclaim_list_size": 524560, "reclaim_list_count": 1, "reclaim_threshold": 1, "allocated_index": 13577836, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 25, "total_records": 1648, "num_rec_allocs": 83165, "num_rec_frees": 81517, "num_rec_swapout": 1347, "num_rec_swapin": 1347, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 290180996, "lss_data_size": 22787656, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 379794266, "est_recovery_size": 28468798, "lss_num_reads": 562, "lss_read_bs": 198042429, "lss_blk_read_bs": 76709888, "lss_rdr_reads_bs": 182333347, "lss_blk_rdr_reads_bs": 64647168, "cache_hits": 18641, "cache_misses": 183, "cache_hit_ratio": 1.00000, "rlss_num_reads": 516, "rcache_hits": 14218, "rcache_misses": 179, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 65.92000, "currSn": 362477, "gcSn": 362474, "gcSnIntervals": "[0 1047 1626 2144 362475]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 21, "num_trim_wctxs": 24, "num_readers": 10, "num_writers": 9, "page_bytes": 21368648, "page_cnt": 42, "page_itemcnt": 326, "avg_item_size": 65548, "avg_page_size": 508777, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 137166328, "page_bytes_marshalled":5155919182, "page_bytes_compressed":2806600285, "compression_ratio": 1.83707, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 15872, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 86581412, "lss_data_size": 55143018, "lss_used_space": 924542610, "lss_disk_size": 951689216, "lss_fragmentation": 94, "lss_num_reads": 22191, "lss_read_bs": 6244867290, "lss_blk_read_bs": 5611159552, "bytes_written": 3695169536, "bytes_incoming": 4851010836, "write_amp": 0.51, "write_amp_avg": 0.56, "lss_gc_num_reads": 18696, "lss_gc_reads_bs": 4695131461, "lss_blk_gc_reads_bs": 4861132800, "lss_gc_status": "frag 93, data: 40859170, used: 673388588, disk: 674747601, relocated: 16907, retries: 492, skipped: 1197, cleaned:2001616763, log:(44836739028 - 45510127616), disk:(44835380015-45510127616), activeFragRatio:30, activeMaxFragRatio:80, run:3 duration:220146 ms elapsed: 220146 ms", "lss_head_offset": 44836739028, "lss_tail_offset": 45510127616, "num_sctxs": 89, "num_free_sctxs": 77, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:36:53 [gsi-test] [] { "memory_quota": 1364653823, "count": 4266, "compacts": 9370, "purges": 0, "splits": 0, "merges": 0, "inserts": 4266, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 8, "persist_conflicts": 0, "memory_size": 126731276, "memory_size_index": 13577868, "allocated": 5731699368, "freed": 5604968092, "reclaimed": 5604443596, "reclaim_pending": 524496, "reclaim_list_size": 524496, "reclaim_list_count": 1, "reclaim_threshold": 1, "allocated_index": 13577868, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1726, "num_rec_allocs": 77714, "num_rec_frees": 75988, "num_rec_swapout": 1668, "num_rec_swapin": 1668, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 279627768, "lss_data_size": 7646752, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 127445866, "est_recovery_size": 28862260, "lss_num_reads": 564, "lss_read_bs": 202029919, "lss_blk_read_bs": 77901824, "lss_rdr_reads_bs": 190960707, "lss_blk_rdr_reads_bs": 67117056, "cache_hits": 17846, "cache_misses": 186, "cache_hit_ratio": 1.00000, "rlss_num_reads": 534, "rcache_hits": 13580, "rcache_misses": 186, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 0.00000, "currSn": 362471, "gcSn": 362465, "gcSnIntervals": "[0 1050 1615 2120 362466]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 22, "num_trim_wctxs": 8, "num_readers": 11, "num_writers": 10, "page_bytes": 32052972, "page_cnt": 61, "page_itemcnt": 489, "avg_item_size": 65548, "avg_page_size": 525458, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 138474536, "page_bytes_marshalled":5661694178, "page_bytes_compressed":3082086368, "compression_ratio": 1.83697, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 13184, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 86581412, "lss_data_size": 52214535, "lss_used_space": 924542610, "lss_disk_size": 951689216, "lss_fragmentation": 94, "lss_num_reads": 22191, "lss_read_bs": 6244867290, "lss_blk_read_bs": 5611159552, "bytes_written": 3695169536, "bytes_incoming": 4853632756, "write_amp": 0.51, "write_amp_avg": 0.56, "lss_gc_num_reads": 18696, "lss_gc_reads_bs": 4695131461, "lss_blk_gc_reads_bs": 4861132800, "lss_gc_status": "frag 93, data: 40859170, used: 673388588, disk: 674747601, relocated: 16907, retries: 492, skipped: 1197, cleaned:2001616763, log:(44836739028 - 45510127616), disk:(44835380015-45510127616), activeFragRatio:30, activeMaxFragRatio:80, run:3 duration:220146 ms elapsed: 220146 ms", "lss_head_offset": 44836739028, "lss_tail_offset": 45510127616, "num_sctxs": 89, "num_free_sctxs": 72, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:36:53 [gsi-test] [] { "memory_quota": 1364653823, "count": 32908, "compacts": 46193, "purges": 0, "splits": 0, "merges": 0, "inserts": 32908, "deletes": 0, "compact_conflicts": 1, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 4, "delete_conflicts": 0, "swapin_conflicts": 9, "persist_conflicts": 0, "memory_size": 70490756, "memory_size_index": 13577820, "allocated": 20556164880, "freed": 20485674124, "reclaimed": 20483313564, "reclaim_pending": 2360560, "reclaim_list_size": 2360560, "reclaim_list_count": 6, "reclaim_threshold": 1, "allocated_index": 13577820, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 868, "num_rec_allocs": 266843, "num_rec_frees": 265975, "num_rec_swapout": 1532, "num_rec_swapin": 1532, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 2157053584, "lss_data_size": 11906295, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 45793442, "est_recovery_size": 30498824, "lss_num_reads": 627, "lss_read_bs": 209466687, "lss_blk_read_bs": 74018816, "lss_rdr_reads_bs": 209466409, "lss_blk_rdr_reads_bs": 74014720, "cache_hits": 620668, "cache_misses": 217, "cache_hit_ratio": 1.00000, "rlss_num_reads": 627, "rcache_hits": 587760, "rcache_misses": 217, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 0.00000, "currSn": 407274, "gcSn": 407262, "gcSnIntervals": "[0 222356 251866 281386 310907 407263]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 4, "num_free_wctxs": 28, "num_trim_wctxs": 16, "num_readers": 16, "num_writers": 9, "page_bytes": 63581560, "page_cnt": 184, "page_itemcnt": 970, "avg_item_size": 65548, "avg_page_size": 345551, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 83414776, "page_bytes_marshalled":50018254264, "page_bytes_compressed":27252969157, "compression_ratio": 1.83533, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 20704, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 6434258, "lss_data_size": 19454659, "lss_used_space": 77556554, "lss_disk_size": 141410304, "lss_fragmentation": 74, "lss_num_reads": 85456, "lss_read_bs": 12527577174, "lss_blk_read_bs": 12583792640, "bytes_written": 11923955712, "bytes_incoming": 2157053584, "write_amp": 1.25, "write_amp_avg": 1.29, "lss_gc_num_reads": 84829, "lss_gc_reads_bs": 12205653811, "lss_blk_gc_reads_bs": 12396666880, "lss_gc_status": "frag 61, data: 18083712, used: 47008936, disk: 48727738, relocated: 367216, retries: 3723, skipped: 32461, cleaned:2737167698, log:(30328873967 - 30376419328), disk:(30327691590-30376419328), activeFragRatio:30, activeMaxFragRatio:80, run:54 duration:282117 ms elapsed: 282135 ms", "lss_head_offset": 30328873967, "lss_tail_offset": 30376419328, "num_sctxs": 37, "num_free_sctxs": 29, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:36:53 [gsi-test] [] { "memory_quota": 1364653823, "count": 25836, "compacts": 168569, "purges": 0, "splits": 0, "merges": 0, "inserts": 25836, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 5, "persist_conflicts": 0, "memory_size": 116372084, "memory_size_index": 13577676, "allocated": 98275039600, "freed": 98158667516, "reclaimed": 98147849816, "reclaim_pending": 10817700, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577676, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1568, "num_rec_allocs": 1330636, "num_rec_frees": 1329068, "num_rec_swapout": 1616, "num_rec_swapin": 1616, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1693498128, "lss_data_size": 1185604, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 19760066, "est_recovery_size": 69520116, "lss_num_reads": 623, "lss_read_bs": 217105340, "lss_blk_read_bs": 75157504, "lss_rdr_reads_bs": 217105340, "lss_blk_rdr_reads_bs": 75157504, "cache_hits": 492772, "cache_misses": 213, "cache_hit_ratio": 1.00000, "rlss_num_reads": 623, "rcache_hits": 466936, "rcache_misses": 213, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 0.00000, "currSn": 405080, "gcSn": 405069, "gcSnIntervals": "[0 19143 30229 40009 405070]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 6, "num_free_wctxs": 33, "num_trim_wctxs": 9, "num_readers": 16, "num_writers": 16, "page_bytes": 357236600, "page_cnt": 692, "page_itemcnt": 5450, "avg_item_size": 65548, "avg_page_size": 516237, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 132965144, "page_bytes_marshalled":11028666818, "page_bytes_compressed":6008563309, "compression_ratio": 1.83549, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 11840, "num_total_smr": 14, "num_active_smr": 3, "num_pending_smr": 11, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 86581412, "lss_data_size": 51095451, "lss_used_space": 924542610, "lss_disk_size": 951689216, "lss_fragmentation": 94, "lss_num_reads": 22191, "lss_read_bs": 6244867290, "lss_blk_read_bs": 5611159552, "bytes_written": 3695169536, "bytes_incoming": 4858090020, "write_amp": 0.51, "write_amp_avg": 0.56, "lss_gc_num_reads": 18696, "lss_gc_reads_bs": 4695131461, "lss_blk_gc_reads_bs": 4861132800, "lss_gc_status": "frag 93, data: 40859170, used: 673388588, disk: 674747601, relocated: 16907, retries: 492, skipped: 1197, cleaned:2001616763, log:(44836739028 - 45510127616), disk:(44835380015-45510127616), activeFragRatio:30, activeMaxFragRatio:80, run:3 duration:220146 ms elapsed: 220146 ms", "lss_head_offset": 44836739028, "lss_tail_offset": 45510127616, "num_sctxs": 89, "num_free_sctxs": 77, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:36:53 [gsi-test] [] { "memory_quota": 1364653823, "count": 25216, "compacts": 164713, "purges": 0, "splits": 0, "merges": 0, "inserts": 25216, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 116896404, "memory_size_index": 13578028, "allocated": 96038419288, "freed": 95921522884, "reclaimed": 95921522884, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13578028, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1576, "num_rec_allocs": 1300476, "num_rec_frees": 1298900, "num_rec_swapout": 1606, "num_rec_swapin": 1606, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1652858368, "lss_data_size": 401087, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 6684783, "est_recovery_size": 54502954, "lss_num_reads": 618, "lss_read_bs": 214777592, "lss_blk_read_bs": 73965568, "lss_rdr_reads_bs": 214384490, "lss_blk_rdr_reads_bs": 72929280, "cache_hits": 483364, "cache_misses": 208, "cache_hit_ratio": 1.00000, "rlss_num_reads": 618, "rcache_hits": 458148, "rcache_misses": 208, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 0.00000, "currSn": 404877, "gcSn": 404867, "gcSnIntervals": "[0 18901 29933 39667 404868]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 4, "num_free_wctxs": 34, "num_trim_wctxs": 4, "num_readers": 16, "num_writers": 16, "page_bytes": 362808180, "page_cnt": 697, "page_itemcnt": 5535, "avg_item_size": 65548, "avg_page_size": 520528, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 130213040, "page_bytes_marshalled":10189834570, "page_bytes_compressed":5551565313, "compression_ratio": 1.83549, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 11712, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 86581412, "lss_data_size": 51095451, "lss_used_space": 924542610, "lss_disk_size": 951689216, "lss_fragmentation": 94, "lss_num_reads": 22191, "lss_read_bs": 6244867290, "lss_blk_read_bs": 5611159552, "bytes_written": 3695169536, "bytes_incoming": 4858090020, "write_amp": 0.51, "write_amp_avg": 0.56, "lss_gc_num_reads": 18696, "lss_gc_reads_bs": 4695131461, "lss_blk_gc_reads_bs": 4861132800, "lss_gc_status": "frag 93, data: 40859170, used: 673388588, disk: 674747601, relocated: 16907, retries: 492, skipped: 1197, cleaned:2001616763, log:(44836739028 - 45510127616), disk:(44835380015-45510127616), activeFragRatio:30, activeMaxFragRatio:80, run:3 duration:220146 ms elapsed: 220146 ms", "lss_head_offset": 44836739028, "lss_tail_offset": 45510127616, "num_sctxs": 89, "num_free_sctxs": 79, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:36:53 [gsi-test] [] { "memory_quota": 1364653823, "count": 8154, "compacts": 5716, "purges": 0, "splits": 0, "merges": 0, "inserts": 8154, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 75616444, "memory_size_index": 13577628, "allocated": 2461616328, "freed": 2385999884, "reclaimed": 2385999884, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 1, "allocated_index": 13577628, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 946, "num_rec_allocs": 31073, "num_rec_frees": 30127, "num_rec_swapout": 1564, "num_rec_swapin": 1564, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 534478392, "lss_data_size": 30837729, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 48183951, "est_recovery_size": 28335004, "lss_num_reads": 577, "lss_read_bs": 195861784, "lss_blk_read_bs": 72658944, "lss_rdr_reads_bs": 186792506, "lss_blk_rdr_reads_bs": 63475712, "cache_hits": 22997, "cache_misses": 185, "cache_hit_ratio": 1.00000, "rlss_num_reads": 551, "rcache_hits": 14843, "rcache_misses": 185, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 0.00000, "currSn": 363118, "gcSn": 363116, "gcSnIntervals": "[0 152518 182031 211556 241076 363117]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 4, "num_free_wctxs": 14, "num_trim_wctxs": 17, "num_readers": 4, "num_writers": 7, "page_bytes": 8783432, "page_cnt": 32, "page_itemcnt": 134, "avg_item_size": 65548, "avg_page_size": 274482, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 91422176, "page_bytes_marshalled":18426771046, "page_bytes_compressed":10039271256, "compression_ratio": 1.83547, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 33704, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 6110610, "lss_data_size": 48352125, "lss_used_space": 76577325, "lss_disk_size": 112148480, "lss_fragmentation": 36, "lss_num_reads": 64800, "lss_read_bs": 9169963586, "lss_blk_read_bs": 9316024320, "bytes_written": 8631414784, "bytes_incoming": 534478392, "write_amp": 2.56, "write_amp_avg": 2.24, "lss_gc_num_reads": 64249, "lss_gc_reads_bs": 8884811213, "lss_blk_gc_reads_bs": 9153712128, "lss_gc_status": "frag 30, data: 33303499, used: 47708927, disk: 49823491, relocated: 552543, retries: 738, skipped: 108718, cleaned:1148173311, log:(10164920410 - 10213490688), disk:(10163667197-10213490688), activeFragRatio:30, activeMaxFragRatio:80, run:214 duration:119923 ms elapsed: 289654 ms", "lss_head_offset": 10164920410, "lss_tail_offset": 10214313984, "num_sctxs": 23, "num_free_sctxs": 15, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:36:53 [gsi-test] [0] Stop and verify... OK 2021-09-13 18:36:53 [gsi-test] [1] Stop and verify... 2021-09-13 18:36:54 [gsi-test] [] { "memory_quota": 1364653823, "count": 6880, "compacts": 16802, "purges": 0, "splits": 0, "merges": 0, "inserts": 6880, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 7, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 126336596, "memory_size_index": 13577708, "allocated": 10296227692, "freed": 10169891096, "reclaimed": 10165957400, "reclaim_pending": 3933696, "reclaim_list_size": 2229096, "reclaim_list_count": 4, "reclaim_threshold": 1, "allocated_index": 13577708, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": -1, "total_records": 1720, "num_rec_allocs": 139907, "num_rec_frees": 138187, "num_rec_swapout": 1584, "num_rec_swapin": 1584, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 450970240, "lss_data_size": 650620, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 10843666, "est_recovery_size": 32403418, "lss_num_reads": 578, "lss_read_bs": 217964494, "lss_blk_read_bs": 91910144, "lss_rdr_reads_bs": 191541695, "lss_blk_rdr_reads_bs": 65396736, "cache_hits": 19317, "cache_misses": 188, "cache_hit_ratio": 1.00000, "rlss_num_reads": 558, "rcache_hits": 12437, "rcache_misses": 188, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 0.00000, "currSn": 362823, "gcSn": 362819, "gcSnIntervals": "[0 1170 1832 2414 362820]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 3, "num_free_wctxs": 12, "num_trim_wctxs": 2, "num_readers": 5, "num_writers": 10, "page_bytes": 38280032, "page_cnt": 73, "page_itemcnt": 584, "avg_item_size": 65548, "avg_page_size": 524384, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 142537760, "page_bytes_marshalled":6165635974, "page_bytes_compressed":3356720146, "compression_ratio": 1.83680, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 11840, "num_total_smr": 3, "num_active_smr": 4, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 86581412, "lss_data_size": 48024909, "lss_used_space": 924542610, "lss_disk_size": 951689216, "lss_fragmentation": 94, "lss_num_reads": 22191, "lss_read_bs": 6244867290, "lss_blk_read_bs": 5611159552, "bytes_written": 3695169536, "bytes_incoming": 4870281948, "write_amp": 0.51, "write_amp_avg": 0.55, "lss_gc_num_reads": 18696, "lss_gc_reads_bs": 4695131461, "lss_blk_gc_reads_bs": 4861132800, "lss_gc_status": "frag 93, data: 40859170, used: 673388588, disk: 674747601, relocated: 16907, retries: 492, skipped: 1197, cleaned:2001616763, log:(44836739028 - 45510127616), disk:(44835380015-45510127616), activeFragRatio:30, activeMaxFragRatio:80, run:3 duration:220146 ms elapsed: 220146 ms", "lss_head_offset": 44836739028, "lss_tail_offset": 45510127616, "num_sctxs": 89, "num_free_sctxs": 71, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:36:54 [gsi-test] [] { "memory_quota": 1364653823, "count": 7464, "compacts": 18131, "purges": 0, "splits": 0, "merges": 0, "inserts": 7464, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 1, "delete_conflicts": 0, "swapin_conflicts": 1, "persist_conflicts": 0, "memory_size": 129877644, "memory_size_index": 13577852, "allocated": 11177037104, "freed": 11047159460, "reclaimed": 11046634964, "reclaim_pending": 524496, "reclaim_list_size": 524496, "reclaim_list_count": 1, "reclaim_threshold": 0, "allocated_index": 13577852, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1774, "num_rec_allocs": 152148, "num_rec_frees": 150374, "num_rec_swapout": 1654, "num_rec_swapin": 1654, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 489250272, "lss_data_size": 6294855, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 104914250, "est_recovery_size": 28337644, "lss_num_reads": 612, "lss_read_bs": 217107994, "lss_blk_read_bs": 75313152, "lss_rdr_reads_bs": 215679028, "lss_blk_rdr_reads_bs": 73871360, "cache_hits": 20852, "cache_misses": 206, "cache_hit_ratio": 1.00000, "rlss_num_reads": 608, "rcache_hits": 13388, "rcache_misses": 206, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 0.00000, "currSn": 362897, "gcSn": 362893, "gcSnIntervals": "[0 1176 1854 2478 362894]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 5, "num_free_wctxs": 21, "num_trim_wctxs": 7, "num_readers": 6, "num_writers": 11, "page_bytes": 35789208, "page_cnt": 67, "page_itemcnt": 546, "avg_item_size": 65548, "avg_page_size": 534167, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 141231456, "page_bytes_marshalled":6990898538, "page_bytes_compressed":3808713501, "compression_ratio": 1.83550, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12864, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 86581412, "lss_data_size": 48024909, "lss_used_space": 924542610, "lss_disk_size": 951689216, "lss_fragmentation": 94, "lss_num_reads": 22191, "lss_read_bs": 6244867290, "lss_blk_read_bs": 5611159552, "bytes_written": 3695169536, "bytes_incoming": 4875394692, "write_amp": 0.51, "write_amp_avg": 0.55, "lss_gc_num_reads": 18696, "lss_gc_reads_bs": 4695131461, "lss_blk_gc_reads_bs": 4861132800, "lss_gc_status": "frag 93, data: 40859170, used: 673388588, disk: 674747601, relocated: 16907, retries: 492, skipped: 1197, cleaned:2001616763, log:(44836739028 - 45510127616), disk:(44835380015-45510127616), activeFragRatio:30, activeMaxFragRatio:80, run:3 duration:220146 ms elapsed: 220146 ms", "lss_head_offset": 44836739028, "lss_tail_offset": 45510127616, "num_sctxs": 89, "num_free_sctxs": 80, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:36:55 [gsi-test] [1] Stop and verify... OK 2021-09-13 18:36:55 [gsi-test] [2] Stop and verify... 2021-09-13 18:36:57 [gsi-test] [2] Stop and verify... OK 2021-09-13 18:36:57 [gsi-test] [3] Stop and verify... 2021-09-13 18:36:58 [gsi-test] [3] Stop and verify... OK 2021-09-13 18:36:58 [gsi-test] [4] Stop and verify... 2021-09-13 18:36:58 [gsi-test] [4] Stop and verify... OK 2021-09-13 18:36:58 [gsi-test] [5] Stop and verify... 2021-09-13 18:36:59 [gsi-test] [5] Stop and verify... OK 2021-09-13 18:36:59 [gsi-test] [6] Stop and verify... 2021-09-13 18:36:59 [gsi-test] [6] Stop and verify... OK 2021-09-13 18:36:59 [gsi-test] [7] Stop and verify... 2021-09-13 18:36:59 [gsi-test] [7] Stop and verify... OK 2021-09-13 18:36:59 [gsi-test] [8] Stop and verify... 2021-09-13 18:37:00 [gsi-test] [8] Stop and verify... OK 2021-09-13 18:37:00 [gsi-test] [9] Stop and verify... 2021-09-13 18:37:03 [gsi-test] [9] Stop and verify... OK 2021-09-13 18:37:03 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper started 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : all deamons started 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : all deamons started 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : all deamons started 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 18:37:03 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper started 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : all deamons started 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : all deamons started 2021-09-13 18:37:03 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper started 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : all deamons started 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : all deamons started 2021-09-13 18:37:03 [gsi-test] [] Stop and verify... OK 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:37:03 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:37:04 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 92, data: 8269608, used: 105988078 log:(59140112734 - 59246833664) head:59140112734 2021-09-13 18:37:04 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 89, data: 2893160, used: 27520534, disk: 29888302, relocated: 8525378, retries: 21072, skipped: 1869175, cleaned:9140424704, log:(68280537438 - 68308320256), disk:(68278431954-68308320256), activeFragRatio:30, activeMaxFragRatio:80, run:276 duration:261447 ms elapsed: 310456 ms 2021-09-13 18:37:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 66, data: 20202620, used: 60104686 log:(1359827014 - 1420791808) head:1358770666 2021-09-13 18:37:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 55, data: 13187040, used: 29625878, disk: 31731304, relocated: 6202407, retries: 2525, skipped: 588285, cleaned:6647251364, log:(8007078378 - 8036704256), disk:(8004972952-8036704256), activeFragRatio:30, activeMaxFragRatio:80, run:280 duration:200475 ms elapsed: 312287 ms 2021-09-13 18:37:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoveryCleaner: starting... frag 69, data: 17973376, used: 58527726 log:(4173037638 - 4231901184) head:4171981290 2021-09-13 18:37:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoveryCleaner: completed... frag 47, data: 16793188, used: 31931926, disk: 33906268, relocated: 6034412, retries: 2496, skipped: 590036, cleaned:6635585956, log:(10808623594 - 10841477120), disk:(10807570852-10841477120), activeFragRatio:30, activeMaxFragRatio:80, run:274 duration:202431 ms elapsed: 312207 ms 2021-09-13 18:37:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 81, data: 14236096, used: 74981358 log:(51786006948 - 51861458944) head:51784954206 2021-09-13 18:37:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 58, data: 12662512, used: 30285334, disk: 31600360, relocated: 7062733, retries: 4493, skipped: 846353, cleaned:7435018170, log:(59221025118 - 59251572736), disk:(59219972376-59251572736), activeFragRatio:30, activeMaxFragRatio:80, run:277 duration:226087 ms elapsed: 312192 ms 2021-09-13 18:37:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: starting... frag 36, data: 39482725, used: 61897558 log:(8473515178 - 8535412736) head:8467698749 2021-09-13 18:37:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: completed... frag 30, data: 32480205, used: 46767440, disk: 49703698, relocated: 447324, retries: 118, skipped: 103770, cleaned:739925040, log:(9213440218 - 9260707840), disk:(9211004142-9260707840), activeFragRatio:30, activeMaxFragRatio:80, run:249 duration:77996 ms elapsed: 303661 ms 2021-09-13 18:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: starting... frag 34, data: 44334499, used: 67240464 log:(7735647728 - 7802888192) head:7730971982 2021-09-13 18:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: completed... frag 30, data: 32444452, used: 46613921, disk: 49051414, relocated: 431935, retries: 26, skipped: 112808, cleaned:737217647, log:(8472865375 - 8519479296), disk:(8470427882-8519479296), activeFragRatio:30, activeMaxFragRatio:80, run:250 duration:75659 ms elapsed: 304673 ms 2021-09-13 18:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: starting... frag 39, data: 40089106, used: 65822702 log:(9016747099 - 9083535360) head:9010460854 2021-09-13 18:37:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: completed... frag 30, data: 33909354, used: 48481151, disk: 49840438, relocated: 557762, retries: 744, skipped: 109227, cleaned:1175791127, log:(10192538226 - 10241269760), disk:(10191429322-10241269760), activeFragRatio:30, activeMaxFragRatio:80, run:215 duration:122651 ms elapsed: 304599 ms 2021-09-13 18:37:12 [gsi-test] [] error: stats.ItemsCount < 0 2021-09-13 18:37:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: starting... frag 86, data: 20585937, used: 152707054 log:(27591706269 - 27745128448) head:27588136136 2021-09-13 18:37:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: completed... frag 72, data: 12762655, used: 46396089, disk: 47755112, relocated: 392340, retries: 3976, skipped: 33683, cleaned:2832876522, log:(30424582791 - 30471229440), disk:(30423474328-30471229440), activeFragRatio:30, activeMaxFragRatio:80, run:56 duration:294021 ms elapsed: 305706 ms 2021-09-13T18:37:22.097-07:00 [Info] memstats {"Alloc":53092816, "TotalAlloc":444115219544, "Sys":283252984, "Lookups":0, "Mallocs":35351122,"Frees":35335423, "HeapAlloc":53092816, "HeapSys":263716864, "HeapIdle":208011264, "HeapInuse":55705600,"HeapReleased":98025472, "HeapObjects":15699,"GCSys":8912896, "LastGC":1631583442074679381,"PauseTotalNs":20450450626, "PauseNs":[4343212, 1385545, 2472361, 1270418, 1453094, 1611064, 890356, 1380525, 4004405, 3652266, 809835, 3715233, 1310567, 5255287, 918048, 1655936, 1930757, 1624337, 1700270, 1658642, 1601503, 1592032, 4089202, 1203331, 1171691, 1584268, 1881853, 1838477, 1797639, 5063137, 3885525, 2020838, 3372285, 6043294, 1875788, 4062489, 2527190, 2697674, 1060030, 1147307, 2680982, 1511530, 1388187, 1543816, 1690576, 1260845, 2989874, 2593688, 1845360, 997267, 2033132, 6777742, 2654855, 1270603, 3822436, 1018405, 3688548, 2200750, 2225833, 1562185, 911717, 3424833, 1446089, 1869932, 1410148, 376042, 1229994, 1449782, 1947757, 7077483, 1664297, 676314, 918150, 1369131, 3953310, 3684812, 2632293, 1197166, 5476088, 1492493, 1254718, 2926441, 1717951, 3811611, 1749476, 994642, 1063789, 1102879, 1942788, 1814181, 1263955, 1863703, 1472033, 1210140, 5005268, 1065324, 833503, 9154842, 1955270, 838687, 1128018, 813484, 1188104, 3403062, 1738509, 6709219, 2144908, 2776966, 1466560, 1078703, 1629420, 2111476, 1887191, 740493, 2966918, 877114, 1836616, 5582889, 984689, 517093, 951749, 1718332, 1067538, 1870011, 3019672, 3416714, 1106618, 3504454, 1327877, 3638311, 462734, 685829, 882875, 2050979, 1235530, 466202, 1617636, 1289914, 1928604, 4068702, 873141, 608220, 1102354, 1391524, 2801879, 876078, 2068660, 3597494, 960505, 1146173, 2166505, 1814440, 775386, 1077488, 1327418, 1841933, 4287481, 1518126, 825238, 1038963, 1287192, 1126320, 1659338, 1559258, 1063199, 1784600, 1494358, 1610731, 4333534, 1762538, 1468285, 1609087, 1524577, 1599785, 1655055, 3060484, 608904, 3165025, 583633, 5150988, 1422554, 2355211, 1851236, 2482033, 3668120, 5615586, 1299157, 1487206, 2086862, 2516147, 1556419, 1734765, 1905459, 7453515, 1528846, 4101056, 4002912, 2818420, 3242969, 796476, 1845342, 908616, 3067858, 1131744, 2895426, 1743598, 1698892, 3490311, 1457294, 1106347, 1140234, 1026537, 1470439, 2029212, 562557, 1390788, 2293729, 1206628, 806714, 2756942, 448384, 1568457, 2656387, 970926, 1566977, 487623, 1309448, 838840, 2645740, 1144401, 1006083, 1641235, 948151, 2094817, 703416, 4628440, 6527177, 790735, 959150, 1604066, 2747484, 5436710, 2890072, 798118, 1755300, 2714483, 1462641, 3721453, 1282381, 420884, 2279970, 7109039, 3996937, 2381508, 2786566, 3910281], "NumGC":7772} Allocated: 1576276152, active: 1577771008, metadata: 13368832 (n_thp 0), resident: 1770496000, mapped: 1783537664, retained: 1290887168 Mon Sep 13 18:37:22 PDT 2021 RSS = 1668136 2021-09-13 18:37:28 [ss-workload] instance inse% dele% read% 2021-09-13 18:37:28 [ss-workload] 0 14% 14% 70% 2021-09-13 18:37:28 [ss-workload] 1 14% 14% 70% 2021-09-13 18:37:28 [ss-workload] 2 14% 14% 70% 2021-09-13 18:37:28 [ss-workload] 3 14% 14% 70% 2021-09-13 18:37:28 [ss-workload] 4 3% 3% 92% 2021-09-13 18:37:28 [ss-workload] 5 3% 3% 92% 2021-09-13 18:37:28 [ss-workload] 6 3% 3% 92% 2021-09-13 18:37:28 [ss-workload] 7 18% 18% 62% 2021-09-13 18:37:28 [ss-workload] 8 18% 18% 63% 2021-09-13 18:37:28 [ss-workload] 9 18% 18% 62% 2021-09-13 18:37:28 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 18:37:28 [ss-diag] 0 18495 17555 1716 1716 86 23 5637128 208 75223332 13577836 1.000000 2.633777 1.000000 0:0:0 2021-09-13 18:37:28 [ss-diag] 1 18668 17740 1794 1794 137 36 8980076 208 74437268 13577820 1.000000 2.516206 1.000000 0:0:0 2021-09-13 18:37:28 [ss-diag] 2 93005 91257 1347 1347 273 33 17894604 208 128173828 13577836 1.000000 0.509545 1.000000 0:0:0 2021-09-13 18:37:28 [ss-diag] 3 87563 85832 1668 1668 193 24 12650764 208 127062704 13577868 1.000000 0.509545 1.000000 0:0:0 2021-09-13 18:37:28 [ss-diag] 4 309379 308543 1532 1532 834 165 54667032 208 68394084 13577820 1.000000 1.247455 1.000000 0:0:0 2021-09-13 18:37:28 [ss-diag] 5 1520551 1518961 1616 1616 3633 480 238135884 208 117815148 13577676 1.000000 0.509545 1.000000 0:0:0 2021-09-13 18:37:28 [ss-diag] 6 1496287 1494685 1606 1606 4172 539 273466256 208 118600796 13578028 1.000000 0.509545 1.000000 0:0:0 2021-09-13 18:37:28 [ss-diag] 7 35200 34260 1564 1564 153 34 10028844 208 75224100 13577628 1.000000 2.555823 1.000000 0:0:0 2021-09-13 18:37:28 [ss-diag] 8 158176 156483 1584 1584 434 58 28447832 208 124572216 13577708 1.000000 0.509545 1.000000 0:0:0 2021-09-13 18:37:28 [ss-diag] 9 172890 171165 1654 1654 574 72 37624552 208 126664952 13577852 1.000000 0.509545 1.000000 0:0:0 2021-09-13T18:37:52.301-07:00 [Info] memstats {"Alloc":113564144, "TotalAlloc":483292249856, "Sys":283252984, "Lookups":0, "Mallocs":38515997,"Frees":38498937, "HeapAlloc":113564144, "HeapSys":262799360, "HeapIdle":146522112, "HeapInuse":116277248,"HeapReleased":48332800, "HeapObjects":17060,"GCSys":9011200, "LastGC":1631583472297827400,"PauseTotalNs":21963164418, "PauseNs":[1928888, 1219713, 1360389, 1627914, 4063029, 1540310, 1926934, 1410966, 875901, 3514135, 988562, 857140, 1708664, 1761693, 3478426, 2313515, 4211961, 3284799, 2164856, 5223461, 2657024, 2528145, 3505601, 2335958, 1087148, 2298329, 1459098, 998549, 4917358, 2923084, 1174356, 2311504, 3763488, 531089, 5066293, 755367, 1630426, 1274258, 4719510, 1467273, 2949661, 1608187, 1056387, 1381411, 1492734, 2455280, 3867881, 1258475, 817573, 1601412, 2693647, 3543255, 3507479, 4136658, 1313314, 1246475, 2599714, 2718035, 968686, 1433981, 5096492, 1337860, 1409888, 5442655, 1060954, 1194802, 954169, 2031734, 6097227, 1079092, 4010465, 1041485, 1852878, 1295125, 1887323, 3740005, 4245299, 3060648, 1161542, 1327384, 745263, 1148988, 4114042, 1502083, 5789022, 2296299, 2548031, 1617814, 1659697, 811045, 1570907, 2745076, 1230324, 805512, 4137476, 3354608, 4157121, 1430488, 4125731, 1423940, 1349838, 1360115, 1276193, 1601942, 2459781, 2542627, 1079292, 945374, 1533843, 6616747, 1264996, 4335300, 1546291, 313500, 1275307, 1674713, 1876890, 1260006, 1745018, 1647921, 1500531, 3195823, 1966014, 748960, 1462750, 894969, 551228, 4306246, 1113292, 1525344, 912629, 944561, 3410984, 1724165, 3772787, 1139904, 4385452, 1278152, 2488918, 1578759, 3041962, 1231518, 977402, 1527243, 1705186, 1701094, 1599244, 1291545, 2780232, 1932756, 3814218, 4492486, 1918485, 3581975, 1153414, 1326728, 870652, 1344629, 1391341, 2513869, 1481303, 921039, 2069063, 1021397, 1566887, 4039356, 1353300, 2256627, 2012497, 1805614, 2325908, 1564332, 3875307, 2383174, 1750935, 3387391, 5204748, 3838543, 3193675, 3400617, 1224587, 1630307, 1715673, 759730, 1400516, 5201806, 1206954, 1421958, 3151148, 2722084, 1335220, 1182888, 4114428, 1424608, 1339859, 3442326, 4277714, 1590236, 967706, 3640696, 1597620, 2707077, 3750559, 5711923, 3178797, 3351383, 1751791, 1647280, 1203601, 2324179, 1446198, 2326072, 1743235, 1791520, 1875986, 1657091, 1748578, 934466, 882376, 1648721, 1171448, 568963, 1688135, 891123, 979705, 1023267, 1333714, 1578621, 4282265, 1304382, 2319719, 1896111, 1536928, 1210348, 2059367, 953399, 2769701, 3065189, 814366, 2879356, 2361234, 1068102, 2476467, 1990848, 2295349, 3857960, 1498648, 2852031, 707714, 1035146, 1639904, 2390359, 2123199, 3569654, 4137588, 801084], "NumGC":8455} Allocated: 1572526200, active: 1574006784, metadata: 13368832 (n_thp 0), resident: 1718161408, mapped: 1731530752, retained: 1342894080 2021-09-13 18:38:10 [gsi-test] [] QuotaTuner: 1364653823 -> 1228188440 2021-09-13 18:38:16 [gsi-test] [] error: stats.ItemsCount < 0 2021-09-13 18:38:18 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: starting... frag 88, data: 83488620, used: 715825092 log:(42835122265 - 43551916032) head:42833973264 2021-09-13 18:38:18 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: completed... frag 89, data: 66567685, used: 660766768, disk: 663773564, relocated: 26033, retries: 771, skipped: 1647, cleaned:2674541695, log:(45509663960 - 46170894336), disk:(45507120772-46170894336), activeFragRatio:30, activeMaxFragRatio:80, run:4 duration:294425 ms elapsed: 305821 ms 2021-09-13T18:38:22.438-07:00 [Info] memstats {"Alloc":93837000, "TotalAlloc":522571569144, "Sys":283252984, "Lookups":0, "Mallocs":41686475,"Frees":41669896, "HeapAlloc":93837000, "HeapSys":262832128, "HeapIdle":166305792, "HeapInuse":96526336,"HeapReleased":116768768, "HeapObjects":16579,"GCSys":8978432, "LastGC":1631583502427164740,"PauseTotalNs":23453146314, "PauseNs":[1504539, 1586337, 1675323, 2990287, 3358393, 4225354, 1984226, 3099393, 3671910, 1559918, 1172008, 5271855, 736194, 1715870, 1155872, 1545010, 1189777, 1826208, 638639, 1417580, 4268844, 6272058, 1433077, 3449833, 1841280, 1281187, 4650481, 1773252, 3001446, 889832, 1718222, 4968218, 1315941, 1917494, 456274, 1296368, 1828362, 1612175, 1327511, 1646123, 1257282, 3918547, 2222645, 1279329, 4418585, 4009976, 1578038, 1828430, 1820237, 688830, 5403918, 4503278, 1861438, 4323913, 1627197, 2420394, 1343755, 1881711, 3656172, 5462949, 2255575, 4482829, 1007792, 1624400, 1089877, 2450744, 1580979, 2304934, 1218044, 1421030, 3256457, 1446524, 3065833, 876965, 2015566, 3625250, 1606723, 1682001, 1629280, 1234157, 3397302, 2215452, 5385147, 1574571, 1077099, 1674208, 1523667, 1002888, 4777853, 4129864, 4700225, 1740653, 1733682, 766200, 4385666, 3654760, 1812559, 1398970, 1559535, 1806755, 6050310, 1135817, 1985671, 1618113, 1609374, 1374815, 5587380, 2885230, 2814809, 963412, 1679904, 1165028, 2411380, 1518520, 1134938, 958827, 1054984, 1760181, 1591679, 1207043, 1442182, 1380692, 2281459, 1685366, 1710554, 3003752, 1254850, 1648091, 3055958, 1487753, 4642024, 4706674, 2782233, 3506763, 1641894, 1162771, 1940577, 2625547, 9029451, 1436697, 1341719, 921389, 1078428, 1315532, 1273365, 2677644, 1190387, 6016091, 990069, 648493, 7652626, 2371275, 1609405, 1582831, 4583770, 3921154, 2839653, 4841986, 2497704, 2648824, 1375925, 4443913, 1434689, 1892594, 1535930, 4089635, 3468067, 4605278, 1549259, 3353522, 3234837, 3145530, 3427989, 5909171, 1641825, 2577640, 2092356, 1779264, 2703466, 6002568, 1093189, 4107619, 1882270, 1215828, 1732558, 1444698, 4830492, 3226892, 3490847, 1053598, 2794358, 1158100, 1605577, 724187, 1098188, 5094626, 2442185, 1216053, 2339062, 1701709, 1897127, 2811221, 1348001, 2528237, 1158475, 1074800, 3487742, 3838285, 1440553, 1047344, 3066205, 1116930, 2637471, 952530, 1822445, 1633485, 868581, 3409585, 1490754, 2885179, 1516123, 3238069, 1415909, 1210826, 939550, 7048376, 1567198, 6082098, 1813100, 779702, 1099212, 1245525, 3308584, 1317191, 3087295, 4429790, 1848019, 1665924, 2546603, 6349758, 1330313, 2555391, 1071227, 3559316, 1508886, 425126, 1262859, 1332283, 4126462, 1811302, 1424419, 2047324, 831970, 1526505, 6011894, 6128185], "NumGC":9134} Allocated: 1569672360, active: 1571188736, metadata: 13368832 (n_thp 0), resident: 1725902848, mapped: 1738944512, retained: 1335480320 Mon Sep 13 18:38:22 PDT 2021 RSS = 1620692 2021-09-13 18:38:33 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Enable page eviction before reaching quota. 2021-09-13 18:38:33 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Enable page eviction before reaching quota. 2021-09-13 18:38:33 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Enable page eviction before reaching quota. 2021-09-13 18:38:40 [ss-workload] instance inse% dele% read% 2021-09-13 18:38:40 [ss-workload] 0 14% 14% 70% 2021-09-13 18:38:40 [ss-workload] 1 14% 14% 70% 2021-09-13 18:38:40 [ss-workload] 2 14% 14% 70% 2021-09-13 18:38:40 [ss-workload] 3 14% 14% 70% 2021-09-13 18:38:40 [ss-workload] 4 3% 3% 92% 2021-09-13 18:38:40 [ss-workload] 5 3% 3% 92% 2021-09-13 18:38:40 [ss-workload] 6 3% 3% 92% 2021-09-13 18:38:40 [ss-workload] 7 18% 18% 62% 2021-09-13 18:38:40 [ss-workload] 8 18% 18% 62% 2021-09-13 18:38:40 [ss-workload] 9 18% 18% 62% 2021-09-13 18:38:40 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 18:38:40 [ss-diag] 0 21972 21044 1716 1716 85 23 5571580 208 74436980 13577836 1.000000 2.643881 1.000000 0:0:0 2021-09-13 18:38:40 [ss-diag] 1 22336 21394 1794 1794 113 30 7406924 208 75355308 13577820 1.000000 2.713259 1.000000 0:0:0 2021-09-13 18:38:40 [ss-diag] 2 110335 108621 1347 1347 337 41 22089676 208 125946396 13577836 1.000000 0.565174 1.000000 0:0:0 2021-09-13 18:38:40 [ss-diag] 3 104745 103057 1668 1668 383 48 25104884 208 124244372 13577868 1.000000 0.565174 1.000000 0:0:0 2021-09-13 18:38:40 [ss-diag] 4 369773 368897 1532 1532 652 127 42737296 208 71017956 13577820 1.000000 1.246809 1.000000 0:0:0 2021-09-13 18:38:40 [ss-diag] 5 1809999 1808411 1616 1616 4171 553 273400708 208 117683460 13577676 1.000000 0.565174 1.000000 0:0:0 2021-09-13 18:38:40 [ss-diag] 6 1779174 1777596 1606 1606 4188 558 274515024 208 117027452 13578028 1.000000 0.565174 1.000000 0:0:0 2021-09-13 18:38:40 [ss-diag] 7 42223 41337 1564 1564 111 28 7275828 208 71682732 13577628 1.000000 2.490633 1.000000 0:0:0 2021-09-13 18:38:40 [ss-diag] 8 190021 188295 1584 1584 499 61 32708452 208 126733644 13577708 1.000000 0.565174 1.000000 0:0:0 2021-09-13 18:38:40 [ss-diag] 9 198686 197016 1654 1654 369 48 24187212 208 123060588 13577852 1.000000 0.565174 1.000000 0:0:0 2021-09-13T18:38:52.590-07:00 [Info] memstats {"Alloc":108076992, "TotalAlloc":560988458176, "Sys":283252984, "Lookups":0, "Mallocs":44853562,"Frees":44836428, "HeapAlloc":108076992, "HeapSys":263159808, "HeapIdle":152395776, "HeapInuse":110764032,"HeapReleased":91439104, "HeapObjects":17134,"GCSys":8978432, "LastGC":1631583532581779345,"PauseTotalNs":24956884809, "PauseNs":[1282877, 3840878, 1103611, 975698, 1671429, 1038944, 1488604, 1375122, 1746016, 4756408, 2512214, 1088830, 881343, 1294487, 1596854, 1879938, 3668266, 1654426, 3336166, 474077, 3302352, 1311870, 3826749, 984934, 4793118, 997534, 1390813, 1432449, 1312857, 1135727, 836848, 972649, 2510463, 608878, 3169646, 3888410, 1665594, 3147048, 1135558, 2160945, 1522394, 2998647, 840067, 551979, 4200046, 450396, 2992274, 1422288, 1730341, 1611232, 1400345, 745293, 991842, 1251497, 1469782, 1472998, 666402, 1163755, 4537390, 4150971, 503470, 1841064, 1473820, 952067, 2138995, 1288069, 1440131, 1875514, 2347721, 2625417, 1216987, 980848, 3047686, 316333, 1438156, 1030742, 1411539, 1257050, 1697325, 3655390, 3590706, 1578362, 3581836, 2874465, 1329054, 5161200, 2906355, 2865864, 1098728, 3186566, 700885, 4822910, 1756651, 1961196, 4140564, 3867879, 4484538, 3647801, 1171787, 1517117, 1074886, 912173, 688609, 1416625, 2134568, 1367524, 1198545, 1139947, 2204954, 1602215, 2677392, 1297744, 1488239, 2389998, 785732, 1279734, 2681695, 1653504, 4122637, 1428731, 1665760, 842571, 805777, 1095887, 1391306, 1282594, 722092, 3669071, 1762654, 3583495, 886590, 1424473, 1848883, 3766580, 1503120, 2077574, 950943, 740710, 1596424, 1332982, 1830772, 2000444, 4415529, 1671516, 3345885, 1931115, 1449011, 5443366, 1440180, 3241107, 1326842, 1821079, 1197181, 1536176, 1389848, 1391989, 3916963, 1139983, 1544811, 2615339, 1353870, 397702, 1759165, 744834, 1644314, 1922782, 1463455, 1882344, 2600081, 1529496, 4508502, 4152254, 2685889, 892231, 5103318, 7074157, 1932327, 1964958, 3949025, 1230903, 1937032, 1710202, 3429764, 966512, 1541265, 1797004, 3453137, 998489, 1545270, 898093, 5247990, 3680587, 1204609, 1617184, 2257706, 3059440, 5887396, 1070237, 1742408, 927337, 1581422, 635055, 1838117, 3860271, 6685263, 2012175, 727228, 542032, 450593, 2292388, 4463307, 2975120, 1228043, 3348740, 1650100, 776080, 1279364, 3392563, 1479737, 958627, 1766169, 3954768, 1345431, 1997618, 1896215, 4837274, 3998547, 3805726, 1783237, 1559195, 3203371, 2283589, 1169994, 861258, 1688562, 1569555, 3256292, 4546774, 1286793, 5101305, 5224565, 2805286, 5768594, 7647123, 1846529, 5248812, 6282545, 5782658, 1090919, 847515, 4171615, 2854271, 1420080, 1227555, 2130047, 3119380], "NumGC":9814} Allocated: 1567982008, active: 1569669120, metadata: 13368832 (n_thp 0), resident: 1746845696, mapped: 1759887360, retained: 1314537472 2021-09-13 18:39:21 [gsi-test] [] error: stats.ItemsCount < 0 2021-09-13T18:39:22.770-07:00 [Info] memstats {"Alloc":156728856, "TotalAlloc":599067766152, "Sys":283252984, "Lookups":0, "Mallocs":47970218,"Frees":47948332, "HeapAlloc":156728856, "HeapSys":263192576, "HeapIdle":103907328, "HeapInuse":159285248,"HeapReleased":92086272, "HeapObjects":21886,"GCSys":9011200, "LastGC":1631583562763527057,"PauseTotalNs":26490515723, "PauseNs":[2650290, 1183484, 4906890, 2934772, 1888588, 1296737, 4139183, 2589911, 630279, 1198780, 2436116, 997807, 1903590, 526279, 1220832, 1445704, 1517495, 955667, 1177194, 2450853, 3861519, 4320060, 2310810, 5280590, 557838, 3653031, 1094838, 1820072, 2116748, 2075419, 1145228, 1801116, 4383200, 1526776, 965992, 4286388, 1867825, 943145, 1925834, 807039, 1404823, 4962637, 4052834, 2537160, 3034895, 1478532, 3640435, 558213, 2658780, 381142, 5088640, 1253761, 1485004, 773404, 1904188, 865011, 1150857, 1184351, 2830783, 1370349, 1530490, 3638452, 1091126, 1411767, 3333213, 3724608, 1359521, 1101956, 1572681, 2185227, 906969, 1623897, 3903774, 1060382, 2294312, 1062505, 4017814, 2225706, 797819, 3861043, 1179406, 1779130, 1845187, 1172246, 1687367, 3828370, 3546713, 1499771, 1007435, 4412808, 803505, 6420167, 4130981, 3466284, 2585708, 3433710, 2531307, 802265, 1978390, 2910505, 1599135, 2510211, 4515554, 1383761, 2036748, 1879876, 931233, 1987629, 1482645, 1791540, 933180, 4256200, 1595780, 1928899, 1492484, 1440217, 748583, 1124387, 1324248, 1207297, 669842, 2089339, 4621762, 1365116, 957716, 1612283, 1099843, 3288807, 6550839, 1561891, 830110, 1223186, 653738, 1741694, 911143, 1236929, 879559, 1585390, 1543798, 1276990, 1932143, 2211914, 1777874, 1253588, 4250729, 1145302, 2894756, 3220030, 5095626, 1275078, 1381492, 740653, 1248710, 1768298, 1692227, 2514691, 4196269, 7960513, 3068028, 2980321, 872135, 983251, 1578824, 3165870, 1286094, 5842937, 1491199, 2128165, 870778, 691099, 4358127, 866159, 1605637, 1806146, 572033, 3648013, 624384, 1854310, 1980065, 1469270, 1465835, 2480431, 3858025, 1066155, 1427060, 1626647, 1518945, 1011535, 1064890, 3473724, 1407993, 1292706, 2964861, 1013037, 1793059, 4364553, 1657159, 3486911, 4383790, 1370845, 1551695, 1314343, 1309700, 1656463, 2574543, 4597366, 1794866, 3265684, 2085909, 3241672, 3232086, 3520109, 2065471, 3992872, 5591282, 906446, 3876798, 1335119, 827387, 2053168, 3956829, 1511032, 2675219, 1492213, 522762, 3355977, 1675793, 4522309, 1399575, 5216146, 1201489, 2139848, 1233107, 803617, 702102, 1144284, 1726766, 586471, 3524169, 557310, 2439932, 837169, 2029545, 3779020, 2195023, 2210457, 935273, 1399484, 8312550, 1827165, 4087605, 1750800, 3444606, 1597635, 1291919, 2271429], "NumGC":10502} Allocated: 1589940296, active: 1591660544, metadata: 13368832 (n_thp 0), resident: 1731055616, mapped: 1744097280, retained: 1330327552 Mon Sep 13 18:39:22 PDT 2021 RSS = 1643548 2021-09-13 18:39:48 [ss-workload] instance inse% dele% read% 2021-09-13 18:39:48 [ss-workload] 0 14% 14% 70% 2021-09-13 18:39:48 [ss-workload] 1 14% 14% 70% 2021-09-13 18:39:48 [ss-workload] 2 14% 14% 70% 2021-09-13 18:39:48 [ss-workload] 3 14% 14% 70% 2021-09-13 18:39:48 [ss-workload] 4 3% 3% 92% 2021-09-13 18:39:48 [ss-workload] 5 3% 3% 92% 2021-09-13 18:39:48 [ss-workload] 6 3% 3% 92% 2021-09-13 18:39:48 [ss-workload] 7 18% 18% 62% 2021-09-13 18:39:48 [ss-workload] 8 18% 18% 62% 2021-09-13 18:39:48 [ss-workload] 9 18% 18% 62% 2021-09-13 18:39:48 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13T18:39:52.994-07:00 [Info] memstats {"Alloc":155685320, "TotalAlloc":638098438728, "Sys":283252984, "Lookups":0, "Mallocs":51177108,"Frees":51157877, "HeapAlloc":155685320, "HeapSys":263815168, "HeapIdle":105553920, "HeapInuse":158261248,"HeapReleased":90554368, "HeapObjects":19231,"GCSys":9011200, "LastGC":1631583592967532445,"PauseTotalNs":27967983206, "PauseNs":[929335, 2520942, 4266642, 2581500, 4038746, 1825725, 1885788, 1853801, 5779221, 1511457, 873141, 3132974, 1186291, 1045790, 2938666, 1129907, 369066, 1070356, 1593229, 4044857, 1602307, 2957715, 1311216, 2141317, 1622436, 680982, 4562088, 5028756, 1297011, 847778, 741323, 718265, 2996004, 3995961, 1657165, 3310386, 1261573, 1414179, 4982093, 1596548, 2862779, 3088823, 3660101, 1632262, 2345076, 2589754, 4419428, 4003657, 3113187, 1180994, 3475481, 751047, 767586, 786123, 2101608, 880943, 3782050, 859491, 2436591, 1030756, 1488746, 1518823, 3932765, 1926172, 1017796, 3291962, 3868655, 2876876, 4133663, 4974489, 3951166, 2032412, 1554000, 1077993, 1734988, 2093340, 2302242, 1527148, 1549696, 4631789, 1398072, 3556835, 2183735, 6349956, 1607559, 1764912, 1144913, 1184970, 1473080, 2702852, 1079938, 3587603, 1576505, 1768790, 2162507, 1265755, 1694987, 1636506, 1223398, 2656183, 1502942, 3889369, 5495278, 1032766, 2986911, 2989121, 1431606, 866139, 717225, 2632120, 1147490, 5392741, 2712950, 3565150, 1687002, 1442979, 1197794, 1374239, 3166812, 1106673, 1556453, 834095, 1774865, 2785312, 502460, 2487536, 1674539, 1634166, 3243270, 1557235, 1124812, 1349030, 1375489, 5334004, 4120389, 2892898, 1364770, 1990227, 931179, 2185154, 4700157, 2456899, 5462759, 1194348, 876189, 3168961, 969215, 3201230, 3626978, 1078316, 870641, 745462, 3813625, 1358405, 5078716, 1035986, 1553596, 3454294, 3022845, 3415385, 2574926, 3162929, 5350053, 1723939, 5431477, 2929083, 2235600, 740461, 2235355, 2898472, 1691892, 2045546, 1693636, 1245519, 1863353, 1569270, 1184659, 840190, 1402845, 3161564, 4499432, 1270759, 1618361, 1619982, 3006904, 1566279, 1868949, 2697057, 2211897, 3118482, 1339196, 2979903, 2135744, 1048063, 1130042, 1650507, 4418224, 1533396, 1368293, 1559953, 5640561, 1596731, 1399877, 1700363, 4685694, 5301572, 818974, 1184248, 929008, 1711621, 941602, 3396461, 3551125, 2505170, 931332, 1757896, 818116, 954048, 8048907, 2289552, 1539691, 1607825, 1089682, 723282, 5488606, 1887827, 1107444, 2355760, 1767737, 1398860, 2214605, 1605638, 365860, 2001896, 830939, 1327070, 3401990, 3661690, 1476908, 6264885, 727178, 991458, 1581917, 3370035, 687782, 2354526, 1625501, 648609, 918100, 2615395, 1126625, 1288132, 4541782, 3314019, 3001823, 1827943], "NumGC":11182} Allocated: 1617842984, active: 1619595264, metadata: 13368832 (n_thp 0), resident: 1746341888, mapped: 1760038912, retained: 1314385920 2021-09-13 18:39:48 [ss-diag] 0 25732 24832 1716 1716 77 23 5047196 208 72602724 13577836 1.000000 2.534692 1.000000 0:0:0 2021-09-13 18:39:48 [ss-diag] 1 25961 25035 1794 1794 103 27 6751444 208 74306412 13577820 1.000000 2.604218 1.000000 0:0:0 2021-09-13 18:39:48 [ss-diag] 2 128593 126905 1347 1347 282 35 18484536 208 124241364 13577836 1.000000 0.634335 1.000000 0:0:0 2021-09-13 18:39:48 [ss-diag] 3 121137 119445 1668 1668 214 26 14027272 208 124507428 13577868 1.000000 0.634335 1.000000 0:0:0 2021-09-13 18:39:48 [ss-diag] 4 429312 428428 1532 1532 738 149 48374424 208 71542884 13577820 1.000000 1.434545 1.000000 0:0:0 2021-09-13 18:39:48 [ss-diag] 5 2100622 2099020 1616 1616 4256 545 278972288 208 118601116 13577676 1.000000 0.634335 1.000000 0:0:0 2021-09-13 18:39:48 [ss-diag] 6 2055083 2053476 1606 1606 4535 579 297260180 208 118928832 13578028 1.000000 0.634335 1.000000 0:0:0 2021-09-13 18:39:48 [ss-diag] 7 48641 47711 1564 1564 157 37 10291036 208 74568252 13577628 1.000000 2.534071 1.000000 0:0:0 2021-09-13 18:39:48 [ss-diag] 8 220293 218543 1584 1584 376 48 24646048 208 128307500 13577708 1.000000 0.634335 1.000000 0:0:0 2021-09-13 18:39:48 [ss-diag] 9 228656 226932 1654 1654 530 67 34740440 208 126600100 13577852 1.000000 0.634335 1.000000 0:0:0 Mon Sep 13 18:40:22 PDT 2021 RSS = 1586688 2021-09-13T18:40:23.175-07:00 [Info] memstats {"Alloc":122837320, "TotalAlloc":676377175136, "Sys":283252984, "Lookups":0, "Mallocs":54295991,"Frees":54278630, "HeapAlloc":122837320, "HeapSys":263618560, "HeapIdle":138133504, "HeapInuse":125485056,"HeapReleased":84418560, "HeapObjects":17361,"GCSys":8978432, "LastGC":1631583623160557574,"PauseTotalNs":29518985138, "PauseNs":[4153834, 2632987, 2043116, 5334836, 1446748, 2673619, 1633957, 3129338, 6288538, 1247752, 662668, 1118760, 1080907, 1030893, 1360171, 886107, 1936300, 1175684, 1182167, 1543226, 1100045, 2163420, 2937117, 1654747, 1935933, 1976490, 1251362, 977448, 3784795, 1170224, 5879567, 1319253, 2096603, 2000095, 4262202, 1710952, 1531624, 3208506, 1406426, 2474141, 744289, 363956, 1246234, 1574477, 486525, 3362431, 3608131, 1181553, 1919287, 1104933, 3024741, 3748158, 3063987, 547937, 1360951, 6070288, 1586279, 1027763, 1938619, 1710080, 932307, 1303927, 879040, 747525, 1854644, 2077177, 2374899, 1636550, 5021535, 1723672, 1367381, 5744598, 4234270, 5391308, 1691086, 1204537, 2180791, 1690006, 5932021, 1530835, 164555, 1253073, 1276691, 1483313, 1200682, 1327906, 2441560, 1616692, 7330456, 1045751, 1440835, 6846665, 877218, 1339814, 1383955, 1779209, 935132, 1003446, 3495983, 3838904, 1168291, 1885054, 6349496, 2451491, 906953, 1172331, 614909, 840434, 2191117, 1471100, 4319876, 1447441, 3215314, 4683351, 1027946, 4434767, 3528872, 1128494, 2892094, 2120976, 1509609, 3205916, 1708406, 7694500, 1438755, 4794548, 1467405, 3463812, 5908441, 2861821, 3739539, 6547991, 1320582, 635135, 1345198, 2779924, 3441676, 1253602, 746590, 1099446, 837809, 1642831, 502984, 1229761, 5230388, 1710880, 1673695, 1376139, 3456545, 2919049, 5146585, 2153498, 2134185, 595012, 1242646, 4314197, 1536660, 1972474, 1068064, 3228339, 3395290, 937237, 1523109, 2400963, 2921203, 6416643, 1548020, 2328296, 902599, 1853498, 1546626, 1180811, 594669, 4566904, 2312969, 3621586, 1023947, 5498745, 1307364, 2004608, 3637941, 4021342, 2608916, 1530049, 1398906, 1315949, 2685823, 1401252, 893635, 576077, 1455854, 1384030, 3590657, 614610, 4768464, 5524603, 3664866, 3509974, 2451725, 1244497, 737560, 998945, 3318148, 1129936, 314110, 1316571, 2686462, 3219121, 3289937, 1516731, 1967011, 1717996, 4543348, 1480671, 2678411, 1618231, 1778437, 3562824, 2895077, 2299613, 1859336, 1533719, 1433979, 931240, 1701134, 1647427, 2474853, 1441412, 3806841, 2071188, 708057, 1273507, 2879996, 1584789, 4174138, 2912036, 1831173, 1076330, 3608969, 4354419, 1283819, 1677313, 1761051, 1837589, 1999205, 1580978, 831625, 3692135, 1798494, 2018364, 4048417, 2617957, 3537653, 1630014, 3404448, 1821519], "NumGC":11865} Allocated: 1550962032, active: 1552658432, metadata: 13397536 (n_thp 0), resident: 1684926464, mapped: 1697968128, retained: 1376456704 2021-09-13T18:40:53.351-07:00 [Info] memstats {"Alloc":121838208, "TotalAlloc":714925544688, "Sys":283252984, "Lookups":0, "Mallocs":57460937,"Frees":57442042, "HeapAlloc":121838208, "HeapSys":263192576, "HeapIdle":138756096, "HeapInuse":124436480,"HeapReleased":100433920, "HeapObjects":18895,"GCSys":9011200, "LastGC":1631583653331663494,"PauseTotalNs":31035509947, "PauseNs":[1143229, 4589285, 2444169, 1025253, 1550045, 3105648, 1098001, 1172720, 1656084, 2020009, 4195117, 1089456, 3568671, 1445293, 1161082, 1366859, 3685919, 1258887, 4200729, 2975049, 1511250, 1115354, 4590751, 4621023, 4661804, 1684026, 1722765, 660745, 1174403, 4005668, 3008549, 1452218, 1738774, 2208605, 2403392, 3031927, 822664, 1454532, 1907878, 1466200, 2028932, 1854569, 5094250, 2183103, 2646842, 2246556, 2131251, 1160273, 959667, 5670486, 5388974, 4088792, 1693126, 978103, 1178171, 3487279, 1553034, 997887, 2785161, 2340968, 4211995, 3304083, 1724348, 1235289, 1751539, 1170681, 2749638, 2715147, 1943031, 1690067, 1765511, 5374510, 1523932, 1444077, 2843858, 1267762, 919157, 1718118, 509915, 1478701, 1247547, 4041581, 2041986, 906109, 2206770, 2362693, 1579171, 559858, 1826678, 1451035, 1133352, 910155, 4145946, 5090860, 1530705, 4277648, 1613235, 887005, 1609918, 1839499, 4267563, 1263807, 2852594, 1422983, 1166254, 4224033, 2147983, 5330855, 1113461, 1967084, 4444508, 2355469, 1742782, 3266925, 1643155, 1411685, 1576323, 817743, 2100343, 2122397, 1260872, 1270762, 1095827, 4868131, 1004152, 2448777, 958915, 4537563, 2953021, 1109716, 1584817, 4530546, 1594959, 1819074, 6828774, 1671279, 1452743, 1488673, 880694, 1076521, 1915635, 1122395, 1732758, 3958670, 4325963, 2378421, 1356734, 1698733, 1889605, 883163, 6377536, 2280182, 1541708, 3199400, 1111958, 690677, 5020307, 1997776, 4953352, 2809935, 3293557, 1736239, 3022916, 2244627, 1732933, 670169, 3871378, 5999258, 4386565, 3862555, 4422390, 3630005, 1976826, 1625720, 2232731, 472529, 1985489, 2686757, 1412398, 1167258, 1271786, 1411074, 3338577, 1614779, 3576208, 2874587, 1870327, 2054139, 3162590, 1347414, 1855273, 918991, 3455306, 5567322, 2321897, 6822930, 3821814, 3663039, 2413124, 1498786, 1145155, 1831370, 5360752, 1843251, 3078606, 2735678, 922790, 4018341, 5204781, 3038251, 1389323, 823731, 1342294, 2700667, 3946788, 2976317, 4679823, 852433, 673125, 1437733, 802693, 3995013, 1132253, 1201395, 7614706, 2939107, 1810037, 9755005, 2032184, 2869584, 2138358, 1000851, 1437862, 5740253, 2197691, 948500, 3797562, 2268896, 3036210, 3848294, 1598629, 2475860, 446720, 3066378, 809729, 1368063, 2565807, 1599131, 1490827, 1600990, 1699496, 883482, 1458448, 951136, 515130, 1222625], "NumGC":12546} Allocated: 1618998584, active: 1620725760, metadata: 13397536 (n_thp 0), resident: 1662423040, mapped: 1676775424, retained: 1397649408 2021-09-13 18:40:56 [ss-workload] instance inse% dele% read% 2021-09-13 18:40:56 [ss-workload] 0 14% 14% 70% 2021-09-13 18:40:56 [ss-workload] 1 14% 14% 70% 2021-09-13 18:40:56 [ss-workload] 2 14% 14% 70% 2021-09-13 18:40:56 [ss-workload] 3 14% 14% 70% 2021-09-13 18:40:56 [ss-workload] 4 3% 3% 92% 2021-09-13 18:40:56 [ss-workload] 5 3% 3% 92% 2021-09-13 18:40:56 [ss-workload] 6 3% 3% 92% 2021-09-13 18:40:56 [ss-workload] 7 18% 18% 62% 2021-09-13 18:40:56 [ss-workload] 8 18% 18% 62% 2021-09-13 18:40:56 [ss-workload] 9 18% 18% 62% 2021-09-13 18:40:56 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 18:40:56 [ss-diag] 0 29188 28280 1716 1716 84 23 5506032 208 73126436 13577836 1.000000 2.376612 1.000000 0:0:0 2021-09-13 18:40:56 [ss-diag] 1 29400 28468 1794 1794 104 24 6816992 208 74697540 13577820 1.000000 2.557486 1.000000 0:0:0 2021-09-13 18:40:56 [ss-diag] 2 145521 143827 1347 1347 279 35 18287892 208 124634508 13577836 1.000000 0.644753 1.000000 0:0:0 2021-09-13 18:40:56 [ss-diag] 3 139832 138144 1668 1668 360 46 23597280 208 124243732 13577868 1.000000 0.644753 1.000000 0:0:0 2021-09-13 18:40:56 [ss-diag] 4 487877 487031 1532 1532 454 98 29758792 208 69050476 13577820 1.000000 1.446755 1.000000 0:0:0 2021-09-13 18:40:56 [ss-diag] 5 2382588 2381004 1616 1616 5018 638 328919864 208 117422004 13577676 1.000000 0.644753 1.000000 0:0:0 2021-09-13 18:40:56 [ss-diag] 6 2343845 2342271 1606 1606 5359 691 351271732 208 116765228 13578028 1.000000 0.644753 1.000000 0:0:0 2021-09-13 18:40:56 [ss-diag] 7 56223 55282 1564 1564 190 40 12454120 208 75289240 13577628 1.000000 2.568771 1.000000 0:0:0 2021-09-13 18:40:56 [ss-diag] 8 252214 250430 1584 1584 521 63 34150508 208 130535828 13577708 1.000000 0.604971 1.000000 0:0:0 2021-09-13 18:40:56 [ss-diag] 9 257602 255940 1654 1654 672 86 44048256 208 122535116 13577852 1.000000 0.604971 1.000000 0:0:0 2021-09-13 18:41:20 [gsi-test] [] QuotaTuner: 1228188440 -> 1105369596 2021-09-13 18:41:21 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 14218 KB (expected = 10240 KB), wCtxCnt = 13, objCnt 39, changed reclaimList flush threshold from 2 to 1, changed reclaimSize flush threshold from 910 KB to 630 KB. Mon Sep 13 18:41:22 PDT 2021 RSS = 1593352 2021-09-13T18:41:23.527-07:00 [Info] memstats {"Alloc":156772984, "TotalAlloc":752870321288, "Sys":283252984, "Lookups":0, "Mallocs":60505499,"Frees":60487371, "HeapAlloc":156772984, "HeapSys":263323648, "HeapIdle":103948288, "HeapInuse":159375360,"HeapReleased":99614720, "HeapObjects":18128,"GCSys":8978432, "LastGC":1631583683515768520,"PauseTotalNs":32611265945, "PauseNs":[3363695, 2534478, 1565612, 2730903, 545756, 1219519, 3779487, 895653, 937664, 3809230, 4337048, 1599771, 1207429, 1075137, 1907949, 926098, 2260886, 2329236, 2219259, 1804413, 627630, 2279287, 1393060, 3384539, 3047464, 5820393, 4699426, 4122730, 752537, 1398341, 1709558, 1944740, 2356213, 1469979, 529900, 2930095, 1841925, 1566611, 1588097, 1267535, 972552, 1329820, 2355237, 4230149, 1923989, 1721748, 1715966, 1750018, 3511475, 1370329, 1188017, 4205673, 2067714, 1179448, 2131557, 4955474, 4750101, 1566791, 2320105, 1554051, 3942995, 1563806, 1342290, 2113118, 1489940, 2133044, 3182213, 1464324, 2993751, 1920797, 1556110, 2115625, 3786192, 2900781, 4114304, 692409, 1397244, 943404, 4324609, 5236000, 423144, 971728, 1947518, 4756050, 1743411, 3894909, 1095426, 1614240, 1429634, 3917286, 1373180, 2063699, 1480586, 2574604, 1623937, 3946603, 6860615, 3243414, 1886321, 3305485, 1070722, 1021525, 1809311, 471391, 3613432, 1941140, 1394356, 4899029, 883493, 2321637, 1103080, 908299, 5131209, 729153, 2773405, 3062408, 1415576, 1512175, 3971631, 1313401, 2728846, 869854, 1043450, 1499271, 1077064, 935052, 2401337, 1668942, 4196786, 7670572, 7093499, 4778580, 6954137, 1149259, 2057445, 5333979, 978514, 2908185, 2135864, 2362011, 1934294, 441191, 1051980, 2679379, 1932800, 5075334, 4447112, 1413845, 5751895, 1635020, 1987143, 3318033, 8490399, 2383990, 3683442, 2444988, 3413495, 3042130, 679087, 3069501, 2415763, 4626907, 1220728, 4714812, 1534147, 1777751, 1396218, 1394627, 1888514, 3353328, 3535631, 6057322, 1783518, 862536, 1407739, 1390751, 1203613, 1075334, 571181, 1580305, 3995252, 3546288, 1249120, 4279631, 4076412, 1735477, 720743, 1257793, 1622365, 1461385, 1600927, 2351546, 1676419, 1413032, 3506186, 669069, 4827162, 6047272, 1148917, 3487267, 1080034, 946009, 1602461, 1326905, 2238120, 1167451, 4238990, 1740774, 1414057, 1237301, 2130538, 3850388, 1018948, 1161194, 2159604, 3963448, 3583418, 4151839, 3889756, 1288344, 1883541, 1807759, 1112689, 1244737, 5489402, 1976912, 2251944, 3343584, 562052, 3159083, 3164091, 1045859, 5029365, 2748359, 1905026, 3398821, 1215812, 3937116, 6985832, 1160186, 8284117, 1027137, 531542, 549577, 916811, 1221721, 1842420, 3324103, 900924, 626396, 1339764, 2810062, 867127, 3055522, 2981753, 3068313], "NumGC":13220} Allocated: 1399192256, active: 1400954880, metadata: 13397536 (n_thp 0), resident: 1659887616, mapped: 1673666560, retained: 1400758272 2021-09-13 18:41:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 200, "num_active_sctxs": 48, "num_free_sctxs": 152, "num_trim_sctxs": 135, "num_wctxs": 251, "num_free_wctxs": 230, "num_active_wctxs": 22, "num_trim_wctxs": 350, "num_readers": 104, "num_writers": 99, "cpu": 9.52162, "total_core": 16, "threads": 48, "assigned_quota": 1105369596, "current_quota": 1105369596, "memory_resident": 0, "memory_jemalloc": 1710694400, "memory_jemalloc_allocated": 1561773616, "memory_in_use": 1711349760, "memory_stats_size_total": 1283958042, "memory_stats_size_global": 1076315540, "memory_stats_size_page": 940594108, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 217952, "reclaim_pending": 64911464, "buf_memused": 142674398, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 26, "num_active_smr": 12, "num_pending_smr": 12, "num_pages": 2080, "items_count": 25, "total_records": 13628, "total_ops": 232631, "compacts": 832968, "purges": 128, "lss_data_size": 375970688, "lss_used_space": 1290518626, "lss_fragmentation": 70, "lss_num_reads": 541137, "lss_read_bs": 80171297634, "lss_blk_read_bs": 81132163072, "rlss_num_reads": 5775, "lss_rdr_reads_bs": 2010432849, "lss_blk_rdr_reads_bs": 707358720, "bytes_written": 75511652352, "bytes_incoming": 15248496788, "write_amp_avg": 1.56, "num_swapper_ctx": 3, "avg_sweep_interval": 284 s, "avg_run_interval": 100 ms, "avg_clock_interval": 2 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 11821, "resident_ratio": 86.74053, "cleaner_num_reads": 535294, "cleaner_read_bs": 77432897235, "cleaner_blk_read_bs": 79708000256, "recovery_num_lssctx": 5, "recovery_num_sctxs": 32, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 12, "recovery_num_trim_sctxs": 12, "recovery_buf_memused": 5242880, "recovery_data_size": 120658560, "recovery_used_space": 384542368, "recovery_lss_blk_read_bs": 62283907072, "recovery_lss_read_bs": 60666172204, "recovery_lss__num_reads": 465688, "recovery_bytes_written": 59818868736, "recovery_lss_frag": 68, "recovery_cleaner_num_reads": 465688, "recovery_cleaner_read_bs": 59978625281, "recovery_cleaner_blk_read_bs": 61597798400, "mvcc_purge_ratio": 545.12000 }2021-09-13 18:41:30 [gsi-test] [5] CreateRecoveryPoint() - DO!!! 2021-09-13 18:41:30 [gsi-test] [6] CreateRecoveryPoint() - DO!!! 2021-09-13 18:41:31 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 18:41:31 [gsi-test] [2] CreateRecoveryPoint() - DO!!! 2021-09-13 18:41:31 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 18:41:31 [gsi-test] [9] CreateRecoveryPoint() - DO!!! 2021-09-13 18:41:31 [gsi-test] [3] CreateRecoveryPoint() - DO!!! 2021-09-13 18:41:31 [gsi-test] [] error: stats.ItemsCount < 0 2021-09-13 18:41:32 [gsi-test] [8] CreateRecoveryPoint() - DO!!! 2021-09-13 18:41:32 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 18:41:32 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13 18:41:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: starting... frag 96, data: 8179756, used: 251154022 log:(7803363492 - 8054910976) head:7800136172 2021-09-13 18:41:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: completed... frag 88, data: 44437772, used: 391187988, disk: 396123832, relocated: 0, retries: 0, skipped: 3330390, cleaned:982761390, log:(8786124882 - 9177968640), disk:(8781844808-9177968640), activeFragRatio:30, activeMaxFragRatio:80, run:289 duration:3397 ms elapsed: 288012 ms 2021-09-13T18:41:53.768-07:00 [Info] memstats {"Alloc":103187608, "TotalAlloc":784438476624, "Sys":283252984, "Lookups":0, "Mallocs":63201740,"Frees":63180897, "HeapAlloc":103187608, "HeapSys":261849088, "HeapIdle":155762688, "HeapInuse":106086400,"HeapReleased":109330432, "HeapObjects":20843,"GCSys":9011200, "LastGC":1631583713753823637,"PauseTotalNs":34248137958, "PauseNs":[589023, 4010105, 1585641, 1653294, 946940, 827270, 1481693, 1442782, 1617834, 1530584, 1537982, 1482735, 3414499, 1715390, 485175, 4057942, 1894316, 942967, 1493870, 4121779, 8615372, 2899038, 1728191, 2627984, 1602555, 2742552, 1977355, 2869656, 2033325, 575976, 1648709, 5841473, 1306968, 413330, 2082331, 1943786, 2294148, 7235500, 1771085, 2176927, 3002279, 7783340, 3730702, 1451556, 1640654, 1394245, 3463274, 2070363, 1243501, 3609606, 2051508, 738029, 1532715, 1339930, 1566352, 5838457, 1140743, 999817, 1670982, 1059623, 2563906, 3774192, 7004757, 7528068, 1243117, 4279277, 1130101, 3271629, 1399166, 1580838, 3817455, 6544675, 5868737, 2123973, 3901570, 3516211, 1195583, 3056758, 4302865, 2822382, 4988214, 4108855, 2610272, 5073935, 1192115, 901971, 756175, 8153620, 2660481, 2412122, 3417071, 2174696, 5190278, 3470303, 2390329, 1927834, 1697900, 3741642, 2948885, 1022850, 929022, 1388023, 1051529, 3477863, 1688102, 5184917, 2077845, 3483588, 2727388, 1904535, 3380233, 5195673, 2318350, 1112827, 3017989, 4003452, 1370832, 1163684, 2637284, 3251694, 1380639, 3641911, 1183276, 1492872, 5995227, 868198, 2880945, 7146875, 1441712, 1267744, 1376440, 5028615, 1576021, 6747847, 1189004, 3429673, 3473409, 1790813, 2787997, 717639, 1760711, 3734068, 2026412, 2022853, 1565952, 1749698, 1753456, 2059142, 5131771, 1044628, 619760, 293957, 5705281, 1635889, 1049658, 853521, 1399175, 1756223, 3857000, 4662119, 812688, 2536837, 3462088, 2651244, 2097243, 1273718, 1398039, 819818, 2171197, 2599510, 1568057, 1105279, 425716, 1092311, 4992875, 2295443, 1482908, 3329633, 1224509, 2737692, 1278237, 1154081, 5331038, 858731, 1184514, 3224505, 1470194, 1002533, 5030913, 1327418, 923323, 3164051, 731906, 7872619, 760703, 7081253, 5699903, 3573965, 953363, 1300155, 1218449, 1018586, 4339147, 4992006, 1308330, 1538571, 1769304, 3995616, 1126506, 3604803, 2800096, 3754456, 1103145, 3923328, 4177774, 537223, 1764584, 3945012, 1338724, 1145725, 5676117, 4028402, 1493013, 1237110, 951530, 7746137, 989184, 2989455, 3936162, 1899384, 1765025, 5761948, 3274524, 2738286, 1344828, 1203827, 1810124, 3495008, 1587864, 2845639, 3611815, 1528281, 2359292, 3181893, 2524694, 1702243, 1254479, 2133094, 8744690, 4965538, 4594581, 1185573, 1986976, 6957805, 1537798, 3481773], "NumGC":13781} Allocated: 1550621576, active: 1552121856, metadata: 22460528 (n_thp 0), resident: 1723494400, mapped: 1738096640, retained: 4299603968 2021-09-13 18:41:56 [gsi-test] [] { "memory_quota": 1105369596, "count": 9044, "compacts": 6142, "purges": 0, "splits": 0, "merges": 0, "inserts": 9044, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 1, "memory_size": 86109212, "memory_size_index": 13577836, "allocated": 2672682044, "freed": 2586572832, "reclaimed": 2586572832, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 2, "allocated_index": 13577836, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1018, "num_rec_allocs": 33322, "num_rec_frees": 32304, "num_rec_swapout": 2617, "num_rec_swapin": 2617, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 592816112, "lss_data_size": 34589156, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 50129211, "est_recovery_size": 28400552, "lss_num_reads": 725, "lss_read_bs": 232280010, "lss_blk_read_bs": 125640704, "lss_rdr_reads_bs": 165670996, "lss_blk_rdr_reads_bs": 61140992, "cache_hits": 38245, "cache_misses": 189, "cache_hit_ratio": 1.00000, "rlss_num_reads": 474, "rcache_hits": 29213, "rcache_misses": 177, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 0.00000, "currSn": 362939, "gcSn": 362935, "gcSnIntervals": "[0 181788 211314 240838 362915 362936]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 21, "num_trim_wctxs": 55, "num_readers": 9, "num_writers": 6, "page_bytes": 12388572, "page_cnt": 32, "page_itemcnt": 189, "avg_item_size": 65548, "avg_page_size": 387142, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 112644621, "page_bytes_marshalled":10996545184, "page_bytes_compressed":5987691780, "compression_ratio": 1.83652, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 37816, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 10197592, "lss_data_size": 53742684, "lss_used_space": 78066744, "lss_disk_size": 172384256, "lss_fragmentation": 31, "lss_num_reads": 113278, "lss_read_bs": 15463398625, "lss_blk_read_bs": 15925334016, "bytes_written": 14948945920, "bytes_incoming": 592816112, "write_amp": 2.58, "write_amp_avg": 2.51, "lss_gc_num_reads": 112792, "lss_gc_reads_bs": 15206751124, "lss_blk_gc_reads_bs": 15775498240, "lss_gc_status": "frag 30, data: 34768131, used: 50021572, disk: 52565352, relocated: 1212507, retries: 253, skipped: 115746, cleaned:716869853, log:(9189735228 - 9239756800), disk:(9187191448-9239756800), activeFragRatio:30, activeMaxFragRatio:80, run:253 duration:70980 ms elapsed: 289146 ms", "lss_head_offset": 9189735228, "lss_tail_offset": 9239756800, "num_sctxs": 35, "num_free_sctxs": 25, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:41:57 [gsi-test] [] { "memory_quota": 1105369596, "count": 9078, "compacts": 6145, "purges": 0, "splits": 0, "merges": 0, "inserts": 9078, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 9, "persist_conflicts": 0, "memory_size": 32007528, "memory_size_index": 13577820, "allocated": 2697743004, "freed": 2665735476, "reclaimed": 2665735476, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 1, "allocated_index": 13577820, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 990, "num_rec_allocs": 33382, "num_rec_frees": 33135, "num_rec_swapout": 3975, "num_rec_swapin": 3232, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 595044744, "lss_data_size": 43016292, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 63259252, "est_recovery_size": 35875600, "lss_num_reads": 974, "lss_read_bs": 304023453, "lss_blk_read_bs": 161746944, "lss_rdr_reads_bs": 227507677, "lss_blk_rdr_reads_bs": 89669632, "cache_hits": 38215, "cache_misses": 303, "cache_hit_ratio": 0.89333, "rlss_num_reads": 673, "rcache_hits": 29161, "rcache_misses": 279, "rcache_hit_ratio": 0.87500, "resident_ratio": 0.24949, "mvcc_purge_ratio": 0.00000, "currSn": 362938, "gcSn": 362937, "gcSnIntervals": "[0 182904 212424 241944 362914 362938]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 21, "num_trim_wctxs": 44, "num_readers": 7, "num_writers": 7, "page_bytes": 8390144, "page_cnt": 29, "page_itemcnt": 128, "avg_item_size": 65548, "avg_page_size": 289315, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 113189638, "page_bytes_marshalled":10815830304, "page_bytes_compressed":5889662892, "compression_ratio": 1.83641, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 18472, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 23416656, "lss_data_size": 66497212, "lss_used_space": 98647376, "lss_disk_size": 227098624, "lss_fragmentation": 32, "lss_num_reads": 112237, "lss_read_bs": 15365417719, "lss_blk_read_bs": 15769133056, "bytes_written": 14815514624, "bytes_incoming": 595044744, "write_amp": 2.15, "write_amp_avg": 2.53, "lss_gc_num_reads": 111539, "lss_gc_reads_bs": 15047575422, "lss_blk_gc_reads_bs": 15593885696, "lss_gc_status": "frag 29, data: 38693266, used: 55226158, disk: 57769731, relocated: 1201678, retries: 285, skipped: 104251, cleaned:714322507, log:(9927762725 - 9983242240), disk:(9925472509-9983242240), activeFragRatio:30, activeMaxFragRatio:80, run:247 duration:68534 ms elapsed: 288904 ms", "lss_head_offset": 9927762725, "lss_tail_offset": 9991376896, "num_sctxs": 36, "num_free_sctxs": 27, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:41:57 [gsi-test] [] { "memory_quota": 1105369596, "count": 8557, "compacts": 18998, "purges": 128, "splits": 30, "merges": 0, "inserts": 8557, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 1, "delete_conflicts": 0, "swapin_conflicts": 2, "persist_conflicts": 1, "memory_size": 108839576, "memory_size_index": 13577836, "allocated": 11704790328, "freed": 11595950752, "reclaimed": 11595950752, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 1, "allocated_index": 13577836, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 25, "total_records": 1596, "num_rec_allocs": 159012, "num_rec_frees": 157597, "num_rec_swapout": 2462, "num_rec_swapin": 2281, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 560894236, "lss_data_size": 31819628, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 353551422, "est_recovery_size": 44609062, "lss_num_reads": 842, "lss_read_bs": 299638986, "lss_blk_read_bs": 118476800, "lss_rdr_reads_bs": 272540932, "lss_blk_rdr_reads_bs": 94937088, "cache_hits": 36315, "cache_misses": 266, "cache_hit_ratio": 0.96857, "rlss_num_reads": 765, "rcache_hits": 27762, "rcache_misses": 262, "rcache_hit_ratio": 0.95736, "resident_ratio": 0.88659, "mvcc_purge_ratio": 63.84000, "currSn": 362760, "gcSn": 362758, "gcSnIntervals": "[0 1626 2144 362735 362759]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 28, "num_trim_wctxs": 46, "num_readers": 10, "num_writers": 9, "page_bytes": 11077612, "page_cnt": 24, "page_itemcnt": 169, "avg_item_size": 65548, "avg_page_size": 461567, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 139991599, "page_bytes_marshalled":5803805384, "page_bytes_compressed":3159517501, "compression_ratio": 1.83693, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 18576, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 108213077, "lss_data_size": 127824015, "lss_used_space": 1450104595, "lss_disk_size": 1485963264, "lss_fragmentation": 91, "lss_num_reads": 38877, "lss_read_bs": 10341436175, "lss_blk_read_bs": 9649094656, "bytes_written": 7920431104, "bytes_incoming": 9393618332, "write_amp": 0.60, "write_amp_avg": 0.62, "lss_gc_num_reads": 34452, "lss_gc_reads_bs": 8450931359, "lss_blk_gc_reads_bs": 8778952704, "lss_gc_status": "frag 88, data: 123661768, used: 1058893228 log:(47484973762 - 48544509952) head:47482681382", "lss_head_offset": 47537426369, "lss_tail_offset": 48593768448, "num_sctxs": 125, "num_free_sctxs": 110, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:41:57 [gsi-test] [] { "memory_quota": 1105369596, "count": 8306, "compacts": 18251, "purges": 0, "splits": 0, "merges": 0, "inserts": 8306, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 8, "persist_conflicts": 6, "memory_size": 98086632, "memory_size_index": 13577868, "allocated": 11151972424, "freed": 11053885792, "reclaimed": 11053885792, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577868, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1576, "num_rec_allocs": 151407, "num_rec_frees": 150128, "num_rec_swapout": 2670, "num_rec_swapin": 2373, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 544441688, "lss_data_size": 34892162, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 387690688, "est_recovery_size": 47297806, "lss_num_reads": 809, "lss_read_bs": 290666186, "lss_blk_read_bs": 109563904, "lss_rdr_reads_bs": 276883608, "lss_blk_rdr_reads_bs": 96030720, "cache_hits": 34675, "cache_misses": 265, "cache_hit_ratio": 0.92720, "rlss_num_reads": 771, "rcache_hits": 26369, "rcache_misses": 265, "rcache_hit_ratio": 0.89840, "resident_ratio": 0.81155, "mvcc_purge_ratio": 0.00000, "currSn": 362759, "gcSn": 362751, "gcSnIntervals": "[0 1615 2120 362734 362752]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 14, "num_free_wctxs": 33, "num_trim_wctxs": 31, "num_readers": 10, "num_writers": 12, "page_bytes": 20516524, "page_cnt": 46, "page_itemcnt": 313, "avg_item_size": 65548, "avg_page_size": 446011, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 137558032, "page_bytes_marshalled":6351335026, "page_bytes_compressed":3457745369, "compression_ratio": 1.83684, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 16592, "num_total_smr": 1, "num_active_smr": 2, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 108417901, "lss_data_size": 126982040, "lss_used_space": 1448926304, "lss_disk_size": 1495699456, "lss_fragmentation": 91, "lss_num_reads": 38939, "lss_read_bs": 10356024443, "lss_blk_read_bs": 9662820352, "bytes_written": 7930167296, "bytes_incoming": 9403319436, "write_amp": 0.60, "write_amp_avg": 0.62, "lss_gc_num_reads": 34504, "lss_gc_reads_bs": 8462018455, "lss_blk_gc_reads_bs": 8791236608, "lss_gc_status": "frag 88, data: 123661768, used: 1058893228 log:(47484973762 - 48544509952) head:47482681382", "lss_head_offset": 47543862572, "lss_tail_offset": 48600674304, "num_sctxs": 125, "num_free_sctxs": 113, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:41:57 [gsi-test] [] { "memory_quota": 1105369596, "count": 65052, "compacts": 99154, "purges": 0, "splits": 0, "merges": 0, "inserts": 65052, "deletes": 0, "compact_conflicts": 5, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 8, "delete_conflicts": 0, "swapin_conflicts": 9, "persist_conflicts": 2, "memory_size": 90150868, "memory_size_index": 13577820, "allocated": 44297319064, "freed": 44207168196, "reclaimed": 44205332260, "reclaim_pending": 1835936, "reclaim_list_size": 262400, "reclaim_list_count": 1, "reclaim_threshold": 1, "allocated_index": 13577820, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1168, "num_rec_allocs": 576287, "num_rec_frees": 575119, "num_rec_swapout": 1545, "num_rec_swapin": 1545, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 4264028496, "lss_data_size": 4616721, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 57709012, "est_recovery_size": 31482366, "lss_num_reads": 632, "lss_read_bs": 210645381, "lss_blk_read_bs": 74715136, "lss_rdr_reads_bs": 210288174, "lss_blk_rdr_reads_bs": 74526720, "cache_hits": 1226128, "cache_misses": 220, "cache_hit_ratio": 1.00000, "rlss_num_reads": 630, "rcache_hits": 1161077, "rcache_misses": 219, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 0.00000, "currSn": 411213, "gcSn": 411194, "gcSnIntervals": "[0 251866 281386 310907 410843 411195]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 34, "num_trim_wctxs": 42, "num_readers": 16, "num_writers": 13, "page_bytes": 217816004, "page_cnt": 588, "page_itemcnt": 3323, "avg_item_size": 65548, "avg_page_size": 370435, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 84463640, "page_bytes_marshalled":55207438176, "page_bytes_compressed":30080996961, "compression_ratio": 1.83529, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 13064, "num_total_smr": 4, "num_active_smr": 5, "num_pending_smr": 1, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 7676881, "lss_data_size": 6854221, "lss_used_space": 93853645, "lss_disk_size": 192278528, "lss_fragmentation": 92, "lss_num_reads": 168139, "lss_read_bs": 24333094802, "lss_blk_read_bs": 24596533248, "bytes_written": 23785984000, "bytes_incoming": 4264028496, "write_amp": 1.45, "write_amp_avg": 1.33, "lss_gc_num_reads": 167508, "lss_gc_reads_bs": 24010171265, "lss_blk_gc_reads_bs": 24408895488, "lss_gc_status": "frag 89, data: 6508416, used: 61038450 log:(33177796750 - 33238835200) head:33174358867", "lss_head_offset": 33191802196, "lss_tail_offset": 33255800832, "num_sctxs": 48, "num_free_sctxs": 37, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:41:57 [gsi-test] [] { "memory_quota": 1105369596, "count": 49914, "compacts": 325972, "purges": 0, "splits": 0, "merges": 0, "inserts": 49914, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 5, "persist_conflicts": 34, "memory_size": 115650912, "memory_size_index": 13577676, "allocated": 189891279956, "freed": 189775629044, "reclaimed": 189772875452, "reclaim_pending": 2753592, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577676, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1566, "num_rec_allocs": 2571428, "num_rec_frees": 2569871, "num_rec_swapout": 1625, "num_rec_swapin": 1616, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 3271762872, "lss_data_size": 1116574, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 12406377, "est_recovery_size": 95496988, "lss_num_reads": 623, "lss_read_bs": 217105340, "lss_blk_read_bs": 75157504, "lss_rdr_reads_bs": 217105340, "lss_blk_rdr_reads_bs": 75157504, "cache_hits": 953152, "cache_misses": 213, "cache_hit_ratio": 1.00000, "rlss_num_reads": 623, "rcache_hits": 903238, "rcache_misses": 213, "rcache_hit_ratio": 1.00000, "resident_ratio": 0.99425, "mvcc_purge_ratio": 0.00000, "currSn": 408774, "gcSn": 408755, "gcSnIntervals": "[0 30229 40009 408442 408756]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 41, "num_trim_wctxs": 46, "num_readers": 16, "num_writers": 16, "page_bytes": 300537580, "page_cnt": 604, "page_itemcnt": 4585, "avg_item_size": 65548, "avg_page_size": 497578, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 130605128, "page_bytes_marshalled":12509288532, "page_bytes_compressed":6815113051, "compression_ratio": 1.83552, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 11784, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 108417901, "lss_data_size": 126530285, "lss_used_space": 1449566348, "lss_disk_size": 1496735744, "lss_fragmentation": 91, "lss_num_reads": 38942, "lss_read_bs": 10356988747, "lss_blk_read_bs": 9663148032, "bytes_written": 7931203584, "bytes_incoming": 9403974916, "write_amp": 0.60, "write_amp_avg": 0.62, "lss_gc_num_reads": 34505, "lss_gc_reads_bs": 8462340143, "lss_blk_gc_reads_bs": 8791236608, "lss_gc_status": "frag 88, data: 123661768, used: 1058893228 log:(47484973762 - 48544509952) head:47482681382", "lss_head_offset": 47544975138, "lss_tail_offset": 48601710592, "num_sctxs": 125, "num_free_sctxs": 113, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:41:57 [gsi-test] [] { "memory_quota": 1105369596, "count": 49610, "compacts": 323619, "purges": 0, "splits": 0, "merges": 0, "inserts": 49610, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 34, "memory_size": 116765804, "memory_size_index": 13578028, "allocated": 188587314500, "freed": 188470548696, "reclaimed": 188465565944, "reclaim_pending": 4982752, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13578028, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1574, "num_rec_allocs": 2553977, "num_rec_frees": 2552403, "num_rec_swapout": 1606, "num_rec_swapin": 1606, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 3251836280, "lss_data_size": 3043089, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 33812100, "est_recovery_size": 81529058, "lss_num_reads": 618, "lss_read_bs": 214777592, "lss_blk_read_bs": 73965568, "lss_rdr_reads_bs": 214384490, "lss_blk_rdr_reads_bs": 72929280, "cache_hits": 949207, "cache_misses": 208, "cache_hit_ratio": 1.00000, "rlss_num_reads": 618, "rcache_hits": 899597, "rcache_misses": 208, "rcache_hit_ratio": 1.00000, "resident_ratio": 1.00000, "mvcc_purge_ratio": 0.00000, "currSn": 408550, "gcSn": 408534, "gcSnIntervals": "[0 29933 39667 408214 408535]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 39, "num_trim_wctxs": 33, "num_readers": 16, "num_writers": 16, "page_bytes": 313057248, "page_cnt": 643, "page_itemcnt": 4776, "avg_item_size": 65548, "avg_page_size": 486869, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 130474816, "page_bytes_marshalled":11397116924, "page_bytes_compressed":6209211530, "compression_ratio": 1.83552, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12224, "num_total_smr": 5, "num_active_smr": 2, "num_pending_smr": 2, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 108417901, "lss_data_size": 126458772, "lss_used_space": 1448528204, "lss_disk_size": 1496735744, "lss_fragmentation": 91, "lss_num_reads": 38946, "lss_read_bs": 10358416698, "lss_blk_read_bs": 9664581632, "bytes_written": 7931203584, "bytes_incoming": 9404499300, "write_amp": 0.60, "write_amp_avg": 0.62, "lss_gc_num_reads": 34508, "lss_gc_reads_bs": 8463446786, "lss_blk_gc_reads_bs": 8792670208, "lss_gc_status": "frag 88, data: 123661768, used: 1058893228 log:(47484973762 - 48544509952) head:47482681382", "lss_head_offset": 47544975138, "lss_tail_offset": 48601710592, "num_sctxs": 125, "num_free_sctxs": 115, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:41:58 [gsi-test] [] { "memory_quota": 1105369596, "count": 15640, "compacts": 11372, "purges": 0, "splits": 0, "merges": 0, "inserts": 15640, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 84398284, "memory_size_index": 13577628, "allocated": 4868778152, "freed": 4784379868, "reclaimed": 4780903944, "reclaim_pending": 3475924, "reclaim_list_size": 2623376, "reclaim_list_count": 8, "reclaim_threshold": 1, "allocated_index": 13577628, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1074, "num_rec_allocs": 62038, "num_rec_frees": 60972, "num_rec_swapout": 1800, "num_rec_swapin": 1792, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 1025170720, "lss_data_size": 34835862, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 50486756, "est_recovery_size": 28072720, "lss_num_reads": 659, "lss_read_bs": 213117810, "lss_blk_read_bs": 85811200, "lss_rdr_reads_bs": 195365888, "lss_blk_rdr_reads_bs": 69287936, "cache_hits": 43984, "cache_misses": 221, "cache_hit_ratio": 1.00000, "rlss_num_reads": 591, "rcache_hits": 28352, "rcache_misses": 213, "rcache_hit_ratio": 1.00000, "resident_ratio": 0.99255, "mvcc_purge_ratio": 0.00000, "currSn": 363425, "gcSn": 363423, "gcSnIntervals": "[0 182031 211556 241076 363398 363424]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 20, "num_trim_wctxs": 32, "num_readers": 5, "num_writers": 8, "page_bytes": 7538020, "page_cnt": 25, "page_itemcnt": 115, "avg_item_size": 65548, "avg_page_size": 301520, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 88222021, "page_bytes_marshalled":20711379524, "page_bytes_compressed":11284317873, "compression_ratio": 1.83541, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 30320, "num_total_smr": 2, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 11743378, "lss_data_size": 53137050, "lss_used_space": 77987298, "lss_disk_size": 133611520, "lss_fragmentation": 31, "lss_num_reads": 127726, "lss_read_bs": 17765164535, "lss_blk_read_bs": 18179297280, "bytes_written": 17242812416, "bytes_incoming": 1025170720, "write_amp": 2.50, "write_amp_avg": 2.38, "lss_gc_num_reads": 127127, "lss_gc_reads_bs": 17470008918, "lss_blk_gc_reads_bs": 18011172864, "lss_gc_status": "frag 30, data: 35085431, used: 50466764, disk: 53049511, relocated: 1679001, retries: 1528, skipped: 109483, cleaned:1219041419, log:(11411579645 - 11462299648), disk:(11409250137-11462299648), activeFragRatio:30, activeMaxFragRatio:80, run:213 duration:120002 ms elapsed: 290437 ms", "lss_head_offset": 11411579645, "lss_tail_offset": 11462299648, "num_sctxs": 32, "num_free_sctxs": 23, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:41:58 [gsi-test] [] { "memory_quota": 1105369596, "count": 13422, "compacts": 32853, "purges": 0, "splits": 0, "merges": 0, "inserts": 13422, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 8, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 0, "memory_size": 111456848, "memory_size_index": 13577708, "allocated": 20089381508, "freed": 19977924660, "reclaimed": 19963828536, "reclaim_pending": 14096124, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577708, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1564, "num_rec_allocs": 273234, "num_rec_frees": 271745, "num_rec_swapout": 2130, "num_rec_swapin": 2055, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 879785256, "lss_data_size": 12001642, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 133351577, "est_recovery_size": 56085124, "lss_num_reads": 727, "lss_read_bs": 275876516, "lss_blk_read_bs": 112627712, "lss_rdr_reads_bs": 249453717, "lss_blk_rdr_reads_bs": 86114304, "cache_hits": 37781, "cache_misses": 241, "cache_hit_ratio": 0.98551, "rlss_num_reads": 707, "rcache_hits": 24359, "rcache_misses": 241, "rcache_hit_ratio": 0.97849, "resident_ratio": 0.95205, "mvcc_purge_ratio": 0.00000, "currSn": 363135, "gcSn": 363132, "gcSnIntervals": "[0 1832 2414 363105 363133]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 10, "num_free_wctxs": 22, "num_trim_wctxs": 19, "num_readers": 4, "num_writers": 12, "page_bytes": 32249616, "page_cnt": 71, "page_itemcnt": 492, "avg_item_size": 65548, "avg_page_size": 454219, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 144657713, "page_bytes_marshalled":6925479620, "page_bytes_compressed":3770602529, "compression_ratio": 1.83670, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 13336, "num_total_smr": 14, "num_active_smr": 3, "num_pending_smr": 13, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 108417901, "lss_data_size": 125032493, "lss_used_space": 1445346835, "lss_disk_size": 1503735808, "lss_fragmentation": 91, "lss_num_reads": 39004, "lss_read_bs": 10371594541, "lss_blk_read_bs": 9676660736, "bytes_written": 7938203648, "bytes_incoming": 9412954992, "write_amp": 0.60, "write_amp_avg": 0.62, "lss_gc_num_reads": 34560, "lss_gc_reads_bs": 8474697006, "lss_blk_gc_reads_bs": 8804098048, "lss_gc_status": "frag 88, data: 123661768, used: 1058893228 log:(47484973762 - 48544509952) head:47482681382", "lss_head_offset": 47551274543, "lss_tail_offset": 48606867456, "num_sctxs": 125, "num_free_sctxs": 113, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:41:58 [gsi-test] [] { "memory_quota": 1105369596, "count": 13672, "compacts": 33382, "purges": 0, "splits": 0, "merges": 0, "inserts": 13672, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 2, "delete_conflicts": 0, "swapin_conflicts": 1, "persist_conflicts": 0, "memory_size": 109817972, "memory_size_index": 13577852, "allocated": 20452447844, "freed": 20342629872, "reclaimed": 20342629872, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577852, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1510, "num_rec_allocs": 278172, "num_rec_frees": 276712, "num_rec_swapout": 2903, "num_rec_swapin": 2853, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 896172256, "lss_data_size": 9435436, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 104838177, "est_recovery_size": 49592886, "lss_num_reads": 949, "lss_read_bs": 338976863, "lss_blk_read_bs": 129327104, "lss_rdr_reads_bs": 324943214, "lss_blk_rdr_reads_bs": 115183616, "cache_hits": 38299, "cache_misses": 320, "cache_hit_ratio": 0.98574, "rlss_num_reads": 910, "rcache_hits": 24627, "rcache_misses": 320, "rcache_hit_ratio": 0.97853, "resident_ratio": 0.96689, "mvcc_purge_ratio": 0.00000, "currSn": 363216, "gcSn": 363213, "gcSnIntervals": "[0 1854 2478 363187 363214]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 28, "num_trim_wctxs": 27, "num_readers": 6, "num_writers": 12, "page_bytes": 27136872, "page_cnt": 63, "page_itemcnt": 414, "avg_item_size": 65548, "avg_page_size": 430744, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 159041079, "page_bytes_marshalled":7671493572, "page_bytes_compressed":4179441996, "compression_ratio": 1.83553, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 13392, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 108417901, "lss_data_size": 125793498, "lss_used_space": 1445523597, "lss_disk_size": 1507307520, "lss_fragmentation": 91, "lss_num_reads": 39016, "lss_read_bs": 10376057176, "lss_blk_read_bs": 9681846272, "bytes_written": 7941775360, "bytes_incoming": 9418657668, "write_amp": 0.60, "write_amp_avg": 0.62, "lss_gc_num_reads": 34572, "lss_gc_reads_bs": 8479159641, "lss_blk_gc_reads_bs": 8809283584, "lss_gc_status": "frag 88, data: 123661768, used: 1058893228 log:(47484973762 - 48544509952) head:47482681382", "lss_head_offset": 47555313405, "lss_tail_offset": 48609517568, "num_sctxs": 125, "num_free_sctxs": 113, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:42:04 [gsi-test] [] Stop and verify... 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 90, data: 3024292, used: 30285334 log:(68280537438 - 68311085056) head:68280537438 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 89, data: 3286556, used: 30416406, disk: 32521890, relocated: 25227591, retries: 73366, skipped: 5705418, cleaned:9184415884, log:(77464953322 - 77495369728), disk:(77462847838-77495369728), activeFragRatio:30, activeMaxFragRatio:80, run:294 duration:243021 ms elapsed: 299783 ms 2021-09-13 18:42:05 [ss-workload] instance inse% dele% read% 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 58, data: 12662512, used: 30285334 log:(59221025118 - 59251572736) head:59219972376 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 30, data: 19153564, used: 27582324, disk: 29953698, relocated: 22784109, retries: 15168, skipped: 2438829, cleaned:7536939148, log:(66757964266 - 66785812480), disk:(66755858782-66785812480), activeFragRatio:30, activeMaxFragRatio:80, run:288 duration:208048 ms elapsed: 300063 ms 2021-09-13 18:42:05 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper stopped 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoveryCleaner: starting... frag 47, data: 17186584, used: 32853526 log:(10808623594 - 10842398720) head:10808623594 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoveryCleaner: completed... frag 30, data: 23480920, used: 33840872, disk: 39042582, relocated: 20635675, retries: 9442, skipped: 1517523, cleaned:6717599534, log:(17526223128 - 17560985600), disk:(17521943018-17560985600), activeFragRatio:30, activeMaxFragRatio:80, run:279 duration:188985 ms elapsed: 300103 ms 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 57, data: 12924776, used: 30547478 log:(8007078378 - 8037625856) head:8006025636 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 35, data: 18170056, used: 28045242, disk: 30351010, relocated: 21035819, retries: 6253, skipped: 1508986, cleaned:6986407936, log:(14993486314 - 15021666304), disk:(14991315294-15021666304), activeFragRatio:30, activeMaxFragRatio:80, run:282 duration:195208 ms elapsed: 300096 ms 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:42:05 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : All daemons stopped 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper stopped 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:42:05 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : All daemons stopped 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper stopped 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:42:05 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : All daemons stopped 2021-09-13 18:42:05 [gsi-test] [0] Stop and verify... 2021-09-13 18:42:05 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:42:06 [gsi-test] [0] Stop and verify... OK 2021-09-13 18:42:06 [gsi-test] [1] Stop and verify... 2021-09-13 18:42:05 [ss-workload] 0 14% 14% 70% 2021-09-13 18:42:05 [ss-workload] 1 14% 14% 70% 2021-09-13 18:42:05 [ss-workload] 2 14% 14% 70% 2021-09-13 18:42:05 [ss-workload] 3 14% 14% 70% 2021-09-13 18:42:05 [ss-workload] 4 3% 3% 92% 2021-09-13 18:42:05 [ss-workload] 5 3% 3% 92% 2021-09-13 18:42:05 [ss-workload] 6 3% 3% 92% 2021-09-13 18:42:08 [gsi-test] [1] Stop and verify... OK 2021-09-13 18:42:08 [gsi-test] [2] Stop and verify... 2021-09-13 18:42:05 [ss-workload] 7 18% 18% 62% 2021-09-13 18:42:05 [ss-workload] 8 18% 18% 62% 2021-09-13 18:42:05 [ss-workload] 9 18% 18% 62% 2021-09-13 18:42:05 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 18:42:05 [ss-diag] 0 35202 34112 2622 2622 197 32 12912956 208 88069180 13577836 1.000000 3.426108 1.000000 0:0:0 2021-09-13 18:42:09 [gsi-test] [2] Stop and verify... OK 2021-09-13 18:42:09 [gsi-test] [3] Stop and verify... 2021-09-13 18:42:05 [ss-diag] 1 35020 34700 4080 4804 221 47 14486108 208 36399156 13577820 0.806167 4.244209 0.306513 0:0:0 2021-09-13 18:42:05 [ss-diag] 2 161799 160354 2446 2587 145 22 9504460 208 110279872 13577836 0.964602 1.103397 0.911097 0:0:0 2021-09-13 18:42:11 [gsi-test] [3] Stop and verify... OK 2021-09-13 18:42:11 [gsi-test] [4] Stop and verify... 2021-09-13 18:42:11 [gsi-test] [4] Stop and verify... OK 2021-09-13 18:42:11 [gsi-test] [5] Stop and verify... 2021-09-13 18:42:12 [gsi-test] [5] Stop and verify... OK 2021-09-13 18:42:12 [gsi-test] [6] Stop and verify... 2021-09-13 18:42:12 [gsi-test] [6] Stop and verify... OK 2021-09-13 18:42:12 [gsi-test] [7] Stop and verify... 2021-09-13 18:42:05 [ss-diag] 3 154444 152993 2581 2702 258 36 16911384 208 109099944 13577868 0.978799 1.103397 0.923028 0:0:0 2021-09-13 18:42:05 [ss-diag] 4 630461 629233 1545 1545 3644 642 238856912 208 94082980 13577820 1.000000 1.606579 1.000000 0:0:0 2021-09-13 18:42:05 [ss-diag] 5 2637501 2635905 1625 1625 5051 654 331082948 208 118207460 13577676 1.000000 1.103397 1.000000 0:0:0 2021-09-13 18:42:05 [ss-diag] 6 2624153 2622588 1606 1606 4982 650 326560136 208 116175224 13578028 1.000000 1.103397 1.000000 0:0:0 2021-09-13 18:42:05 [ss-diag] 7 66105 65051 1940 2028 179 36 11733092 208 83602652 13577628 0.971875 3.012907 0.922942 0:0:0 2021-09-13 18:42:05 [ss-diag] 8 279623 277980 2121 2130 397 57 26022556 208 121554904 13577708 0.990164 1.103397 0.994552 0:0:0 2021-09-13 18:42:05 [ss-diag] 9 284311 282763 2908 2926 499 68 32708452 208 115192148 13577852 0.992157 1.103397 0.988506 0:0:0 2021-09-13 18:42:14 [gsi-test] [7] Stop and verify... OK 2021-09-13 18:42:14 [gsi-test] [8] Stop and verify... 2021-09-13 18:42:16 [gsi-test] [8] Stop and verify... OK 2021-09-13 18:42:16 [gsi-test] [9] Stop and verify... 2021-09-13 18:42:16 [gsi-test] [9] Stop and verify... OK 2021-09-13 18:42:16 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper started 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : all deamons started 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : all deamons started 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : all deamons started 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : all deamons started 2021-09-13 18:42:16 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper started 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : all deamons started 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : all deamons started 2021-09-13 18:42:16 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper started 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : all deamons started 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : all deamons started 2021-09-13 18:42:16 [gsi-test] [] Stop and verify... OK 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Starting daemon 2021-09-13 18:42:16 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Starting daemon Mon Sep 13 18:42:23 PDT 2021 RSS = 1696708 2021-09-13T18:42:23.960-07:00 [Info] memstats {"Alloc":117563992, "TotalAlloc":821993468152, "Sys":283252984, "Lookups":0, "Mallocs":66442747,"Frees":66422822, "HeapAlloc":117563992, "HeapSys":262832128, "HeapIdle":142426112, "HeapInuse":120406016,"HeapReleased":94748672, "HeapObjects":19925,"GCSys":9011200, "LastGC":1631583743942655833,"PauseTotalNs":35764099950, "PauseNs":[2800980, 1867039, 6455259, 1473481, 1044317, 1606868, 1094376, 1532153, 1449010, 1155700, 6568062, 5191748, 3506594, 5492902, 2642357, 5283055, 1749840, 2201490, 7002040, 1689733, 5851768, 3455644, 3513172, 3004287, 5858549, 4682974, 1495686, 4103719, 3209026, 5132812, 8999756, 3791815, 1490745, 5446340, 5028849, 1493719, 5971683, 8696883, 2227871, 2036038, 2291672, 5487871, 2797142, 4429219, 3947615, 4668927, 2846731, 3495533, 1812527, 2935822, 1086108, 1365124, 1828570, 1940032, 1294452, 1011752, 2677292, 3709021, 2204211, 1182378, 3912638, 1675495, 6430458, 5265266, 6412988, 4593061, 2538261, 4085815, 2622881, 6434630, 8063884, 4756448, 4922854, 9847588, 3747743, 7865790, 6833492, 4795526, 1259529, 1412254, 1452191, 3139876, 7420548, 7687830, 7372726, 4819947, 4170438, 1346003, 1525471, 3382562, 5397757, 5518844, 1796339, 4101056, 1164981, 2328317, 1644295, 593860, 1323740, 1707271, 2737406, 1493782, 1242946, 349431, 959619, 1428022, 2161403, 649211, 1252791, 1931774, 1823608, 1333729, 1579131, 1409418, 1226064, 1494108, 801749, 931371, 3487693, 1402180, 812522, 1269638, 964748, 1299681, 1698370, 1282029, 1498300, 1535538, 1940924, 545275, 1724588, 428799, 2406567, 1473269, 5273393, 743815, 1064386, 1009217, 1385894, 1475835, 1979531, 1468001, 1344184, 1419782, 1792433, 1373477, 1194891, 1267909, 1561823, 1474031, 1847909, 494389, 1551015, 1063397, 1466400, 1903993, 1166565, 1413135, 1500555, 1301179, 1055345, 1283253, 1512554, 1211560, 1553753, 1007691, 1082996, 1490608, 1625307, 1824900, 2056457, 1339906, 1365423, 1432009, 1113273, 895631, 513699, 1844230, 1021708, 1143716, 1205221, 2434429, 2141133, 1737305, 1435179, 1767719, 2286888, 1844577, 1519676, 1514563, 1362839, 1784899, 1406293, 564077, 3125886, 1639594, 1614536, 792340, 1763732, 1590721, 1207149, 1194827, 1750262, 977859, 799632, 804118, 1427706, 1510614, 1108210, 1250352, 937179, 3021310, 2389493, 1220678, 1549445, 1698627, 1405539, 1449235, 1545785, 2675768, 2661022, 1309945, 1099554, 3178305, 762340, 1733886, 1299762, 1556387, 469104, 1811342, 1961102, 2999372, 1404858, 1238496, 1720666, 1622125, 1335138, 1267646, 2010064, 698469, 1182819, 1798951, 1210248, 1554325, 443787, 2492229, 2720693, 2421904, 1517302, 1891147, 3845396, 1582754, 3451987, 3582621, 4803192, 3159793], "NumGC":14432} Allocated: 1686434496, active: 1687949312, metadata: 22460528 (n_thp 0), resident: 1866039296, mapped: 1880641536, retained: 4157059072 2021-09-13 18:42:25 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: starting... frag 72, data: 12762655, used: 46396089 log:(30424582791 - 30471229440) head:30423474328 2021-09-13 18:42:25 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : logCleaner: completed... frag 91, data: 5471058, used: 65175237, disk: 67894082, relocated: 1160837, retries: 12845, skipped: 37963, cleaned:2904811002, log:(33329393793 - 33395068928), disk:(33327174846-33395068928), activeFragRatio:30, activeMaxFragRatio:80, run:61 duration:296508 ms elapsed: 308912 ms 2021-09-13 18:42:25 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: starting... frag 33, data: 32480455, used: 48481151 log:(10192538226 - 10241269760) head:10191429322 2021-09-13 18:42:25 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : logCleaner: completed... frag 31, data: 45257095, used: 66532877, disk: 69148364, relocated: 1731740, retries: 1573, skipped: 112424, cleaned:1307471443, log:(11500009669 - 11566792704), disk:(11497644340-11566792704), activeFragRatio:30, activeMaxFragRatio:80, run:218 duration:131645 ms elapsed: 318327 ms 2021-09-13 18:42:26 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: starting... frag 32, data: 31480297, used: 46767440 log:(9213440218 - 9260707840) head:9211004142 2021-09-13 18:42:26 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : logCleaner: completed... frag 30, data: 47906661, used: 69144607, disk: 72221157, relocated: 1247487, retries: 307, skipped: 107938, cleaned:823443231, log:(10036883449 - 10106777600), disk:(10034556443-10106777600), activeFragRatio:30, activeMaxFragRatio:80, run:254 duration:81843 ms elapsed: 319925 ms 2021-09-13 18:42:26 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: starting... frag 31, data: 32444452, used: 47474081 log:(8472865375 - 8520339456) head:8471683293 2021-09-13 18:42:26 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : logCleaner: completed... frag 37, data: 43118574, used: 69274865, disk: 76867547, relocated: 1277235, retries: 265, skipped: 121227, cleaned:793418174, log:(9266283549 - 9336455168), disk:(9259587621-9336455168), activeFragRatio:30, activeMaxFragRatio:80, run:263 duration:82197 ms elapsed: 319662 ms 2021-09-13 18:42:36 [plasma] [i] Plasma: mvccPurger: starting... purge_ratio 65.60000 2021-09-13 18:42:37 [plasma] [i] Plasma: mvccPurger: finished... purge_ratio 56.64000 2021-09-13 18:42:48 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 10629 KB (expected = 10240 KB), wCtxCnt = 20, objCnt 24, changed reclaimList flush threshold from 6 to 0, changed reclaimSize flush threshold from 409 KB to 409 KB. 2021-09-13T18:42:54.121-07:00 [Info] memstats {"Alloc":120680208, "TotalAlloc":855951831216, "Sys":283252984, "Lookups":0, "Mallocs":69587657,"Frees":69566698, "HeapAlloc":120680208, "HeapSys":263061504, "HeapIdle":139640832, "HeapInuse":123420672,"HeapReleased":94789632, "HeapObjects":20959,"GCSys":9043968, "LastGC":1631583774120796759,"PauseTotalNs":37647713753, "PauseNs":[605093, 970005, 1043839, 2580333, 4320740, 5409529, 1216572, 4942530, 4505343, 2032008, 776506, 3496570, 6291415, 4493011, 4852588, 4096618, 2255644, 1762866, 6023799, 1513585, 3245645, 2120920, 1467089, 4427993, 712768, 641805, 4527969, 3053036, 2102480, 1312810, 2529685, 3725989, 762023, 1632317, 1118022, 3286091, 2611575, 4084143, 4621882, 2653335, 2225011, 1672805, 6898873, 3728593, 3128196, 2344853, 4876657, 2581389, 4948855, 2282421, 1768778, 3533180, 3066731, 4805831, 736257, 2621776, 4003737, 2674597, 4813734, 5709848, 1522522, 3725114, 3897284, 1296763, 1081585, 4440667, 3115435, 1589426, 1839283, 2661568, 2797542, 3569382, 3223661, 3711011, 1365100, 547693, 1635182, 2439257, 1138591, 5640472, 1506327, 6007855, 11653282, 7350713, 4167185, 4158023, 8094836, 8992639, 7327602, 1285611, 8956728, 5434144, 4956506, 4951650, 7592529, 5648050, 1391563, 3821970, 4859154, 7611811, 5147982, 1182950, 2332191, 1395284, 4353834, 1536330, 4573519, 4214239, 5409760, 3814615, 1637086, 3770223, 7820266, 2674062, 4458594, 7679482, 5562210, 3790400, 3896867, 3356681, 553443, 3328260, 3717101, 2503719, 3810795, 1812704, 1629021, 4298179, 1532089, 2821931, 8749198, 1002498, 4825396, 6388817, 3400432, 2498735, 8403062, 1449089, 3394868, 2146274, 4557318, 3821940, 5539579, 3418872, 5635415, 2449727, 2644577, 1535949, 2117663, 4276972, 3328605, 2069821, 2068594, 5267182, 1520459, 4343449, 1562441, 5214212, 2707506, 5267050, 2921617, 1255582, 3356348, 1001402, 5760605, 2985281, 1371235, 1536457, 3961305, 3215228, 1660325, 3766132, 1858744, 4163287, 4364203, 704727, 7600710, 3720672, 5261616, 4884763, 5146571, 4032728, 1343630, 5031543, 5252942, 3250982, 1240311, 1354983, 2023407, 2199753, 1824964, 2544883, 5583452, 4596736, 1626399, 1431883, 4414264, 968012, 5148938, 5194165, 1644922, 4945578, 2013275, 1168578, 1261061, 3581477, 1836184, 798999, 4169228, 1775057, 1172772, 804933, 6302402, 1043290, 3138523, 1213928, 5033079, 706887, 2094996, 1424022, 3116876, 6403971, 2224510, 1032849, 1663229, 3712882, 487850, 6757122, 5057082, 3498689, 5571895, 4179105, 4696844, 3177592, 4268010, 2364676, 1388899, 1417217, 1799322, 3259621, 1216364, 1907844, 6262892, 1099814, 2564223, 3460062, 1063922, 1621021, 584056, 2703746, 1607421, 787229, 3901712, 4718021, 2040969, 1719504], "NumGC":15025} Allocated: 1693376720, active: 1694908416, metadata: 22460528 (n_thp 0), resident: 1879592960, mapped: 1894195200, retained: 4143505408 2021-09-13 18:43:13 [ss-workload] instance inse% dele% read% 2021-09-13 18:43:13 [ss-workload] 0 14% 14% 70% 2021-09-13 18:43:13 [ss-workload] 1 14% 14% 70% 2021-09-13 18:43:13 [ss-workload] 2 14% 14% 70% 2021-09-13 18:43:13 [ss-workload] 3 14% 14% 70% 2021-09-13 18:43:13 [ss-workload] 4 3% 3% 92% 2021-09-13 18:43:13 [ss-workload] 5 3% 3% 92% 2021-09-13 18:43:13 [ss-workload] 6 3% 3% 92% 2021-09-13 18:43:13 [ss-workload] 7 18% 18% 62% Mon Sep 13 18:43:23 PDT 2021 RSS = 1700888 2021-09-13T18:43:24.218-07:00 [Info] memstats {"Alloc":79255624, "TotalAlloc":883918335968, "Sys":283252984, "Lookups":0, "Mallocs":72288576,"Frees":72269240, "HeapAlloc":79255624, "HeapSys":263290880, "HeapIdle":181280768, "HeapInuse":82010112,"HeapReleased":158015488, "HeapObjects":19336,"GCSys":9011200, "LastGC":1631583804161668189,"PauseTotalNs":39297825969, "PauseNs":[4155964, 1057835, 2503328, 2896986, 849526, 1642336, 2267273, 1584479, 4755779, 3023195, 1034619, 2485427, 3675244, 5209156, 2250763, 4250072, 2939231, 4160610, 1906151, 1864000, 3097745, 5502342, 1183085, 1022917, 890947, 1667282, 1853333, 2551889, 2989359, 4578732, 1136456, 1376912, 7187855, 1324543, 3729190, 9141078, 2430307, 694814, 2643021, 2300048, 4455684, 1268332, 1754717, 2243321, 324596, 5712100, 4328432, 1434020, 1725503, 696638, 2105816, 1421671, 2504412, 1854706, 3138919, 1704123, 3353752, 4801574, 8796421, 5952846, 3414934, 3526466, 5758383, 4014020, 1240237, 3818005, 3150373, 1242746, 3380288, 1009437, 1672308, 5316271, 5687155, 1301921, 1594483, 439004, 5279509, 5238586, 3194570, 1760258, 2428018, 897557, 6655931, 4411586, 6322537, 3743464, 1696616, 1387109, 1031370, 1808654, 4737033, 1515185, 3665772, 1461691, 5306453, 2447254, 5210835, 4140716, 4761199, 4162276, 1401699, 2212901, 3413860, 6236963, 10073145, 2614140, 5014664, 4062989, 3852675, 3884943, 4836024, 2597178, 3360465, 498153, 824916, 2912850, 417623, 4405015, 1295089, 6414952, 6274328, 3899977, 4934221, 3657180, 375132, 3884187, 4896746, 3178028, 3251530, 3095305, 898639, 3666206, 4552182, 1701229, 2176110, 4241625, 2790165, 945589, 5145238, 6337483, 3268448, 1086826, 4140958, 4225269, 3429451, 3816111, 9605701, 6703197, 3696427, 4493603, 1035223, 3846322, 1184081, 3610823, 461833, 581758, 2162757, 1090863, 3036345, 905947, 3351694, 991589, 650159, 6028729, 1324880, 3165160, 2549907, 3604007, 1405558, 8156472, 3495217, 1797500, 1000964, 1485936, 1545638, 1564568, 1509466, 3394060, 6323728, 3513199, 1575636, 5239082, 2957619, 4586283, 1577149, 1521149, 1647649, 2632409, 1963766, 4048211, 4520890, 3056278, 1633699, 4885477, 3711091, 2740927, 945876, 3053502, 3211280, 2906873, 2692759, 886953, 3625765, 1061668, 5822645, 8200059, 4800640, 4576475, 1188553, 4403529, 1756062, 902702, 802205, 2200430, 5277351, 4078595, 4349346, 1913388, 1433960, 3463997, 1519305, 2232921, 1034716, 5334787, 6534300, 5053491, 5616962, 5258838, 2106511, 3309979, 4909770, 6431123, 2361416, 6161698, 2007378, 4161448, 2331686, 1664799, 1590729, 1284979, 1531757, 2892638, 1918494, 2804735, 2170319, 4753252, 4781220, 691501, 4708693, 3251669, 2807069, 3870722, 3135882, 984514, 6340414, 1491212], "NumGC":15529} Allocated: 1696089288, active: 1697640448, metadata: 22460528 (n_thp 0), resident: 1760182272, mapped: 1774784512, retained: 4262916096 2021-09-13 18:43:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: starting... frag 89, data: 66567685, used: 660766768 log:(45509663960 - 46170894336) head:45507120772 2021-09-13 18:43:36 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : logCleaner: completed... frag 62, data: 386927076, used: 1044837591, disk: 1052918651, relocated: 70026, retries: 2187, skipped: 3153, cleaned:3164250995, log:(48673914955 - 49719144448), disk:(48666225797-49719144448), activeFragRatio:30, activeMaxFragRatio:80, run:5 duration:305890 ms elapsed: 318069 ms 2021-09-13T18:43:54.275-07:00 [Info] memstats {"Alloc":72854824, "TotalAlloc":894369527056, "Sys":283252984, "Lookups":0, "Mallocs":73577758,"Frees":73559882, "HeapAlloc":72854824, "HeapSys":263520256, "HeapIdle":187768832, "HeapInuse":75751424,"HeapReleased":149897216, "HeapObjects":17876,"GCSys":8912896, "LastGC":1631583834147209089,"PauseTotalNs":40036247373, "PauseNs":[1618774, 2328984, 570237, 2571444, 386713, 2003054, 4952822, 3511692, 369142, 1382857, 1539457, 964875, 846110, 8179613, 977614, 3779748, 1349336, 1579839, 3581948, 3792224, 4759148, 1307822, 429811, 4183836, 1881627, 517782, 3353308, 9677390, 770947, 11709207, 524823, 1263425, 2100257, 3800213, 4309700, 1874238, 7666067, 3999106, 1556706, 4120315, 466136, 3934825, 2988597, 535273, 2837387, 2912904, 353770, 464636, 4093655, 790989, 703634, 3294609, 417130, 5631235, 5615993, 2928200, 4529824, 8706532, 3277791, 2332304, 3597292, 2598431, 3906052, 961757, 1158052, 3710898, 4507589, 5134626, 4954437, 2401466, 1439556, 4188361, 935965, 3105034, 2741924, 930916, 4313983, 439483, 4341553, 1078670, 1621331, 4848248, 2527154, 729564, 4937006, 3427891, 5880399, 1862524, 7234232, 5305952, 3445159, 6311363, 4187561, 736635, 3217063, 4871601, 5750832, 7479331, 6232025, 5616203, 2426245, 405937, 6779795, 3820296, 4743561, 1383000, 4970129, 1704965, 532741, 2910165, 9527176, 7556691, 3517718, 6838644, 3331389, 3245892, 3822078, 5833338, 927641, 5960447, 2584822, 3910377, 5348784, 4684964, 965038, 3843788, 4967328, 6120405, 1760037, 5677506, 351952, 4044983, 4137571, 902984, 1226707, 8437948, 802245, 1862112, 3903699, 4073968, 3442907, 518418, 4191871, 5000468, 8065289, 2138125, 7124260, 3582327, 2889313, 5920365, 7420066, 3712800, 4548811, 1159039, 1736331, 928540, 5986824, 5400217, 1027695, 2611516, 4449674, 369032, 3719474, 4117565, 2862009, 5531288, 5081204, 6163223, 4620687, 6802405, 1700454, 4975400, 768460, 4259835, 3474092, 602274, 1662570, 927334, 2898032, 4359423, 4218084, 4484963, 4189847, 3443633, 3430909, 3925817, 6800279, 11346482, 964439, 569185, 5918708, 9238269, 5183115, 7820967, 1108999, 3022403, 1575564, 1401935, 432471, 746073, 3058461, 1055239, 2149814, 2299056, 2465930, 1296201, 599624, 5786159, 2023950, 1142684, 376915, 493378, 1194901, 1347483, 1450428, 970351, 2671753, 947793, 490211, 586368, 593366, 3368658, 3544997, 5457618, 660262, 397225], "NumGC":15755} Allocated: 1625714296, active: 1627254784, metadata: 22460528 (n_thp 0), resident: 1743110144, mapped: 1758171136, retained: 4279529472 2021-09-13 18:43:13 [ss-workload] 8 18% 18% 62% 2021-09-13 18:43:13 [ss-workload] 9 18% 18% 62% 2021-09-13 18:43:13 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 18:44:17 [gsi-test] [] error: stats.ItemsCount < 0 2021-09-13 18:43:13 [ss-diag] 0 55611 54447 6982 6989 167 27 10946516 208 97217758 13577836 1.000000 3.824765 0.994355 0:0:0 2021-09-13 18:43:13 [ss-diag] 1 53985 52764 11156 11156 191 32 12519668 208 99543320 13577820 1.000000 5.109751 1.000000 0:0:1 2021-09-13 18:43:13 [ss-diag] 2 179515 178882 4360 5190 210 27 13765080 208 69390655 13577836 0.881356 0.893931 0.493284 0:0:0 2021-09-13 18:43:13 [ss-diag] 3 169940 169708 4860 6288 208 28 13633984 208 30497284 13577868 0.641509 0.893931 0.139759 0:0:0 2021-09-13 18:43:13 [ss-diag] 4 1005697 1004545 1703 1703 2823 518 185042004 208 89103092 13577820 1.000000 3.630602 1.000000 0:0:0 2021-09-13 18:43:13 [ss-diag] 5 2889107 2887624 3158 3188 410 58 26874680 208 113335793 13577676 0.942918 0.893931 0.980892 0:0:0 2021-09-13 18:43:13 [ss-diag] 6 2888094 2886501 2029 2040 615 82 40312020 208 118011504 13578028 1.000000 0.893931 0.993142 0:0:0 2021-09-13 18:43:13 [ss-diag] 7 99738 98488 5357 5357 404 65 26481392 208 96063548 13577628 1.000000 3.960453 1.000000 0:0:1 2021-09-13 18:43:13 [ss-diag] 8 307063 305451 3763 3786 167 21 10946516 208 127869102 13577708 1.000000 0.893931 0.986819 0:0:0 2021-09-13 18:43:13 [ss-diag] 9 315607 314222 4502 4744 249 32 16321452 208 113515912 13577852 0.933333 0.893931 0.854916 0:0:0 Mon Sep 13 18:44:23 PDT 2021 RSS = 1836428 2021-09-13T18:44:24.362-07:00 [Info] memstats {"Alloc":131621128, "TotalAlloc":911016014392, "Sys":283252984, "Lookups":0, "Mallocs":75318251,"Frees":75291997, "HeapAlloc":131621128, "HeapSys":263290880, "HeapIdle":129187840, "HeapInuse":134103040,"HeapReleased":92930048, "HeapObjects":26254,"GCSys":8978432, "LastGC":1631583864342118437,"PauseTotalNs":40870571167, "PauseNs":[726107, 384002, 3957026, 1549493, 4219714, 1228890, 4636520, 1307310, 3292921, 1423504, 2331807, 3520958, 1220808, 439896, 1729937, 491280, 436902, 376576, 1035801, 3373373, 704073, 1822644, 4802992, 892702, 753575, 3457073, 2331413, 854132, 1500867, 3250537, 5545486, 468736, 919117, 5515952, 974441, 811031, 589212, 3395503, 2939648, 1745228, 3473577, 813489, 2444951, 403933, 3950911, 2114538, 5298763, 440783, 654487, 389575, 683969, 1632227, 1251104, 5831463, 528513, 1462359, 928044, 499995, 1182610, 1026329, 312541, 1287402, 8694687, 4320431, 8833547, 3399487, 3197711, 5185434, 3586271, 7254071, 7134299, 5963410, 5593438, 935657, 4544236, 4428373, 5453781, 1979676, 2239538, 1438037, 7418178, 2486750, 4812077, 7633345, 7050789, 7288451, 3698208, 4959648, 6625692, 6188640, 1853333, 1046260, 3250520, 2373248, 9767850, 4040461, 3874210, 2616235, 8431844, 4900512, 4222531, 9968527, 7987668, 5710262, 4760039, 988558, 4762431, 3352626, 9531548, 4647398, 3311916, 2424655, 1656651, 1776797, 902721, 1071639, 1223664, 1581101, 1447461, 1142141, 3072686, 5220438, 2842702, 4960591, 3363946, 2872862, 2833413, 3843198, 1212367, 2180562, 1210736, 1560351, 920008, 1415056, 989791, 934272, 1760135, 1396631, 2293299, 2144149, 7525526, 4812072, 5293255, 3680219, 4588898, 3722031, 4422153, 761482, 3660314, 2708972, 2915337, 1001729, 5639745, 1033922, 2537259, 1023711, 3175280, 1174505, 853596, 2059522, 2766054, 1850780, 5996893, 4459049, 1283143, 3424953, 6807177, 2115165, 2273177, 2045285, 1445568, 1274151, 1500376, 6101418, 4678925, 1974950, 765163, 4205793, 1345788, 1336277, 2013012, 1385945, 2474430, 1456539, 5916968, 6862278, 1451767, 3714256, 2458718, 3405290, 3495107, 4701126, 4057515, 5404189, 1464261, 3933016, 1307096, 711083, 838146, 4322190, 1792471, 704884, 5121321, 1159537, 3573010, 6875205, 961850, 4975906, 3122672, 1437294, 3970213, 2115248, 3660459, 799536, 816950, 1418322, 456712, 586680, 2458823, 3983041, 326597, 686590, 3175606, 508227, 827689, 980054, 885426, 6780822, 857928, 6346389, 1000250, 1021610, 1377222, 1655921, 523649, 2339883, 3092681, 931970, 1680574, 655222, 2345047, 447840, 1745511, 535025, 968775, 456536, 720519, 433342, 2424443, 1162620, 3734462, 470913, 2631322, 1972682, 2884083, 3224846], "NumGC":16085} Allocated: 1753345376, active: 1754931200, metadata: 22460528 (n_thp 0), resident: 1943867392, mapped: 1959387136, retained: 4078313472 2021-09-13 18:44:31 [gsi-test] [] QuotaTuner: 1105369596 -> 994832636 2021-09-13 18:44:34 [plasma] [i] Plasma: SMR reclaim pending is higher than expected: pending = 17543 KB (expected = 9715 KB), wCtxCnt = 24, objCnt 30, changed reclaimList flush threshold from 0 to 0, changed reclaimSize flush threshold from 356 KB to 323 KB. 2021-09-13T18:44:54.489-07:00 [Info] memstats {"Alloc":98569200, "TotalAlloc":945085757064, "Sys":283252984, "Lookups":0, "Mallocs":78487361,"Frees":78466077, "HeapAlloc":98569200, "HeapSys":262799360, "HeapIdle":161431552, "HeapInuse":101367808,"HeapReleased":109543424, "HeapObjects":21284,"GCSys":8912896, "LastGC":1631583894463465333,"PauseTotalNs":42802921183, "PauseNs":[3267906, 3210333, 5167900, 5350016, 2984940, 5897649, 5169644, 2510513, 7421768, 7080001, 7843723, 2020246, 4837820, 1014091, 4574442, 6166660, 3143058, 1438821, 2213824, 1707702, 2426476, 750806, 739528, 2950625, 3958746, 753390, 1164658, 4960066, 928444, 3788496, 762076, 1440056, 1242019, 4160428, 5677486, 583684, 1448794, 3384009, 3689346, 6341450, 3857015, 953051, 2958705, 2656959, 8752737, 3471545, 2284050, 7432715, 1408388, 1633941, 867038, 1109834, 1636724, 6237790, 7221683, 1439042, 591948, 3625795, 2950200, 1292967, 1754980, 5182596, 1244381, 907961, 3039474, 7016230, 1813326, 7406638, 6170773, 5248282, 1965801, 1614506, 4408886, 4631857, 707554, 1420393, 7553488, 5114277, 4451291, 5300261, 2511575, 1664152, 5586267, 1003372, 4541802, 1051325, 3994968, 1493580, 1746747, 296828, 7780722, 1461548, 2378657, 5685668, 568812, 839193, 2233917, 4851708, 5028660, 1578144, 3176722, 1420104, 3753922, 2234864, 1201805, 1302517, 8150734, 8890351, 4253376, 958743, 1565440, 6657049, 4846521, 4765914, 603479, 1484919, 1368980, 1475446, 4248068, 2019006, 1022166, 2299562, 3922977, 1296982, 4039567, 4868163, 4927334, 3341003, 1091267, 3542392, 4441729, 3765801, 3150615, 7191342, 6375662, 2351179, 2870405, 5232304, 1795334, 3350520, 3528003, 2310474, 5765037, 4017584, 2244654, 1897981, 2313462, 3658932, 821392, 1067332, 8455459, 3416673, 6538782, 1456368, 4100473, 2182975, 4563200, 1667484, 1701455, 2472937, 1215322, 3731138, 2630978, 8006731, 6201259, 1878470, 470983, 2879368, 3383978, 1089466, 1435216, 1978694, 3960915, 4664848, 3329751, 1193961, 2241333, 2508800, 8157259, 2022147, 1513912, 1084759, 1194798, 2122202, 1310641, 8665305, 3411346, 568228, 4313279, 999386, 1480906, 1267044, 457331, 2076262, 867896, 4668320, 1175563, 4492582, 2414562, 1816251, 5860682, 7070308, 7407530, 6515248, 1654986, 4108558, 4272858, 6017077, 4534892, 6607067, 1498108, 4800120, 4199693, 3283832, 1703350, 1773412, 6687799, 7530031, 3688321, 1399394, 5509780, 1553211, 4217392, 1298885, 3520654, 7339781, 1486724, 1021128, 7140612, 3613987, 866398, 5175549, 1805102, 1005640, 1777217, 2096921, 5301805, 6242371, 2209834, 3544827, 1606933, 393630, 3196864, 1685490, 3051018, 1278341, 1261182, 988199, 1358746, 6464936, 5308725, 1065854, 1422778, 2464372, 2899147, 5592387], "NumGC":16689} Allocated: 1657120528, active: 1658736640, metadata: 22460528 (n_thp 0), resident: 1827467264, mapped: 1842069504, retained: 4195631104 2021-09-13 18:45:20 [ss-workload] instance inse% dele% read% 2021-09-13 18:45:20 [ss-workload] 0 14% 14% 70% 2021-09-13 18:45:20 [ss-workload] 1 14% 14% 70% 2021-09-13 18:45:20 [ss-workload] 2 14% 14% 70% 2021-09-13 18:45:20 [ss-workload] 3 14% 14% 70% 2021-09-13 18:45:20 [ss-workload] 4 3% 3% 92% 2021-09-13 18:45:20 [ss-workload] 5 3% 3% 92% 2021-09-13 18:45:20 [ss-workload] 6 3% 3% 92% Mon Sep 13 18:45:23 PDT 2021 RSS = 1644548 2021-09-13 18:45:20 [ss-workload] 7 18% 18% 62% 2021-09-13 18:45:20 [ss-workload] 8 18% 18% 62% 2021-09-13 18:45:20 [ss-workload] 9 18% 18% 62% 2021-09-13 18:45:20 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13T18:45:24.728-07:00 [Info] memstats {"Alloc":130678280, "TotalAlloc":977865598032, "Sys":283252984, "Lookups":0, "Mallocs":81535383,"Frees":81515931, "HeapAlloc":130678280, "HeapSys":263421952, "HeapIdle":129949696, "HeapInuse":133472256,"HeapReleased":109027328, "HeapObjects":19452,"GCSys":8978432, "LastGC":1631583924725858461,"PauseTotalNs":44955700675, "PauseNs":[1592212, 2033656, 7683713, 2930599, 2208761, 4889063, 2370440, 2830246, 4855881, 795719, 5954758, 5074327, 4283712, 1889148, 3170816, 4221101, 6846351, 5470639, 5596022, 3805849, 7525668, 3779967, 2389566, 2257096, 6829025, 3893082, 4045649, 830579, 3272353, 3638692, 3126836, 2311999, 3483245, 2581614, 4926335, 1505081, 4742799, 4821647, 1674605, 4644267, 1093327, 3457703, 1386609, 9790313, 4555635, 3095826, 4023566, 3817847, 5649212, 7176760, 4579614, 3250091, 7159666, 754898, 3396872, 1453506, 1142198, 7459783, 3446220, 1153519, 5500484, 1257436, 5441544, 1100876, 604174, 6023646, 782964, 500580, 6966461, 3021205, 3210669, 3222305, 4122470, 1975733, 7437579, 3234625, 5791865, 1417163, 5247920, 3583325, 6426420, 2187141, 2530961, 6658890, 5104401, 10766184, 6787397, 1285456, 2699695, 1301669, 3895137, 3767947, 2695376, 1821021, 4465519, 3574673, 4401619, 855315, 7348619, 6751722, 1197434, 4142579, 3690134, 5793374, 3193625, 1254436, 3048803, 8622384, 1098469, 4964457, 7835750, 2940519, 5959874, 1231562, 818208, 1059526, 3646885, 3195623, 1166732, 1831219, 2346383, 1098575, 615940, 1238098, 3999825, 5162316, 4298990, 3126971, 9120537, 4676035, 2413113, 2491592, 3421539, 4374181, 5639820, 3522392, 1405936, 2743357, 4288260, 4741238, 3981814, 832235, 4040348, 3562855, 6061118, 3059221, 6581980, 3788946, 3292420, 8077247, 6592228, 2318828, 3368704, 2272717, 4184122, 4118619, 1972856, 2544287, 4203151, 1130085, 1411085, 1536136, 4398040, 6246288, 3755400, 913023, 5283210, 4064741, 4228569, 7472500, 5400542, 1470956, 4416093, 6125227, 1949750, 3024887, 4566217, 7832203, 1663467, 3083802, 2471062, 2511500, 6914812, 1319830, 8184444, 5304983, 5916897, 5170343, 4185844, 1837227, 5297549, 2388781, 1810875, 4412884, 7283441, 3507679, 3444663, 1618532, 3850968, 5269140, 2131435, 825385, 1807461, 6835346, 1697145, 4396807, 4237771, 3739095, 5922923, 4817645, 5051454, 5321307, 4346287, 637229, 1369118, 6018729, 2170654, 1306537, 2746266, 1159081, 5961550, 6033894, 6669949, 1354189, 4412797, 4679510, 1240987, 2652527, 3251703, 2086280, 3585159, 2860824, 1994969, 5456302, 2958264, 2664354, 3979803, 1369455, 6992884, 2091543, 3049754, 2544117, 6226651, 4655735, 7463853, 5224539, 5911869, 1175154, 7128800, 7345779, 5770197, 2922013, 1881852, 4841163, 6590556, 794801], "NumGC":17270} Allocated: 1594473000, active: 1596014592, metadata: 22460528 (n_thp 0), resident: 1674964992, mapped: 1691140096, retained: 4346560512 2021-09-13 18:45:20 [ss-diag] 0 67589 67228 11341 12238 346 58 22679608 208 39086832 13577836 0.892857 5.733861 0.286963 0:0:0 2021-09-13 18:45:25 [gsi-test] [] error: stats.ItemsCount < 0 2021-09-13 18:45:20 [ss-diag] 1 67562 67532 17289 18522 369 61 24187212 208 15810884 13577820 0.872222 6.527079 0.023753 0:0:0 2021-09-13 18:45:20 [ss-diag] 2 194704 193560 7394 7926 351 44 23007348 208 93174156 13577836 0.938053 0.808242 0.682578 0:0:1 2021-09-13 18:45:20 [ss-diag] 3 186105 185207 7979 8745 542 70 35527016 208 73632188 13577868 0.951070 0.808242 0.539663 0:0:1 2021-09-13 18:45:20 [ss-diag] 4 1217394 1216225 1917 1922 3041 529 199331468 208 90347504 13577820 1.000000 1.885029 0.995741 0:0:0 2021-09-13 18:45:20 [ss-diag] 5 3159881 3158282 3242 3251 4451 574 291754148 208 118404264 13577676 1.000000 0.808242 0.994403 0:0:1 2021-09-13 18:45:20 [ss-diag] 6 3168014 3166449 2159 2168 5203 653 341046244 208 116175296 13578028 0.999897 0.808242 0.994282 0:0:1 2021-09-13 18:45:20 [ss-diag] 7 119612 118831 10389 10821 483 78 31659684 208 69902472 13577628 0.794953 4.733421 0.643858 0:0:0 2021-09-13 18:45:20 [ss-diag] 8 339332 337895 6578 6891 625 75 40967500 208 108835744 13577708 0.948905 0.808242 0.821143 0:0:0 2021-09-13 18:45:20 [ss-diag] 9 345474 344267 7148 7623 488 61 31987424 208 94545048 13577852 0.940133 0.808242 0.717598 0:0:0 2021-09-13T18:45:54.939-07:00 [Info] memstats {"Alloc":99733608, "TotalAlloc":1009919949552, "Sys":283252984, "Lookups":0, "Mallocs":84548988,"Frees":84529513, "HeapAlloc":99733608, "HeapSys":263127040, "HeapIdle":160473088, "HeapInuse":102653952,"HeapReleased":119529472, "HeapObjects":19475,"GCSys":9011200, "LastGC":1631583954926959636,"PauseTotalNs":47005339395, "PauseNs":[4267439, 4662106, 2401060, 4535004, 7909459, 4499614, 4099861, 6801814, 7139716, 4224883, 1968177, 7525007, 3106592, 5914728, 5218788, 5836943, 3824191, 5589822, 4929504, 3995901, 760630, 1641067, 4367537, 5314194, 3998479, 6083835, 6021451, 4933218, 5312936, 3025255, 7847089, 4740159, 5484395, 5056861, 4582739, 1625064, 3115163, 1750137, 5028662, 1034463, 2709819, 2360102, 9906891, 3289361, 798307, 8618014, 8308845, 9083165, 1067412, 5814224, 8162783, 7798705, 1290364, 6094899, 5295857, 8329212, 3006957, 5447357, 7434728, 962021, 5045328, 6457844, 4651451, 8348995, 3261117, 2202738, 8064693, 1759834, 3230378, 5704124, 1954292, 1262588, 5377161, 6942645, 5597952, 2007558, 6374283, 1167746, 6317069, 7777687, 8187648, 3529919, 1923927, 4449240, 5968015, 1947857, 1549088, 9716967, 5133220, 4274070, 4051274, 4459020, 735188, 6426265, 1052579, 2704386, 2014405, 6925750, 7193181, 7288461, 2580118, 2093476, 6475443, 1450467, 3138318, 6715821, 5254290, 7802838, 3500158, 3622685, 4276711, 4363060, 2221964, 2574556, 3392674, 4928911, 3661105, 6711204, 6219007, 4587671, 4540571, 4883766, 1404530, 1399993, 6075565, 4985266, 4978376, 5181785, 2132014, 7120395, 5077010, 9442459, 4965070, 4831707, 1632639, 5843570, 1172185, 1176541, 843393, 2858786, 4894988, 3023853, 5925311, 2532064, 5021294, 4515682, 2516597, 1241385, 2303421, 3598380, 2535293, 3126894, 1472083, 4259408, 6166197, 3890532, 4854024, 1568971, 3272821, 4329391, 4741820, 2971832, 2542602, 3281287, 5139555, 7041879, 3504919, 1691893, 4454187, 1675052, 539682, 5073186, 6253337, 3814005, 2964518, 3704108, 909861, 868624, 3952195, 4136335, 2254915, 2754412, 1888537, 2604652, 638654, 3998030, 4652602, 7271889, 3669984, 2618888, 5908671, 3865244, 6062006, 922857, 1895096, 4612057, 1148525, 7513614, 1762321, 3803522, 1140420, 2086802, 2534595, 1463395, 1602903, 3370541, 2649427, 2844081, 4288759, 4851479, 1541909, 1090203, 1681970, 1109420, 2863142, 1675796, 6746000, 3471781, 3741400, 1072313, 3697808, 3536442, 5334428, 4384980, 2405573, 3923239, 2275690, 5988754, 1641038, 1160193, 7181685, 3563724, 2670712, 4023147, 1361363, 1968045, 4556011, 4814693, 4260953, 8112026, 4850166, 3690720, 2816593, 1593671, 822508, 1331423, 5927078, 5432726, 1181596, 1279661, 2606994, 2130625, 3041441, 3196037, 8324918, 4987986], "NumGC":17838} Allocated: 1680730992, active: 1682276352, metadata: 22460528 (n_thp 0), resident: 1757028352, mapped: 1771630592, retained: 4266070016 Mon Sep 13 18:46:23 PDT 2021 RSS = 1667640 2021-09-13T18:46:25.127-07:00 [Info] memstats {"Alloc":117606992, "TotalAlloc":1041660612312, "Sys":283252984, "Lookups":0, "Mallocs":87497845,"Frees":87476645, "HeapAlloc":117606992, "HeapSys":262406144, "HeapIdle":142090240, "HeapInuse":120315904,"HeapReleased":133332992, "HeapObjects":21200,"GCSys":9011200, "LastGC":1631583985121212126,"PauseTotalNs":49080407461, "PauseNs":[1345743, 3145241, 1822185, 2697514, 4116888, 5583312, 4343781, 3676619, 3619378, 946776, 4518639, 1358242, 3835197, 1544562, 805338, 1896261, 3442644, 7801352, 5426556, 2146888, 8473252, 454123, 873201, 4215903, 3510792, 6360300, 2882776, 1112196, 3247165, 4559150, 3316586, 1608173, 1182367, 4767111, 5651084, 5324051, 4941971, 2410879, 1878707, 2420436, 932973, 2625358, 8333118, 1501566, 3432933, 3387350, 3782185, 7250116, 1136211, 3235722, 1202880, 2692620, 4241936, 1255546, 6108318, 8014501, 1073019, 2380960, 1892021, 451703, 4062523, 957981, 2602755, 1449323, 1171232, 7684173, 7462530, 2594487, 2458587, 4544858, 4235256, 5406718, 4944428, 1138655, 7160056, 6071522, 1392443, 2366935, 1200025, 4420882, 7047751, 2753478, 3664503, 1442308, 1603208, 4699995, 4445354, 1270906, 5052559, 3789189, 2275695, 7030922, 2620992, 3597838, 2481874, 1522726, 3024002, 4572982, 1415312, 8021969, 2355218, 7304708, 7050719, 3459643, 7675937, 5661132, 2115969, 4114809, 2448330, 8645714, 5139479, 5632905, 582604, 2991795, 998202, 6885924, 4245826, 3533254, 3517282, 4923635, 4475740, 6256302, 3071120, 2492969, 4691808, 893799, 4870648, 4060863, 1332661, 7044636, 4754207, 875975, 5439526, 2673878, 5485337, 3250742, 3990674, 1191066, 1315947, 4996861, 8022879, 4068146, 5305537, 2746023, 2501320, 4282086, 5459253, 1275359, 7816339, 7274077, 1313388, 3234563, 1762003, 1155928, 5557628, 1896076, 1821109, 7881973, 6647630, 4149369, 2577002, 2626992, 5424556, 3243367, 3029859, 3163707, 1326648, 6333697, 3935000, 2873869, 4469710, 2740576, 2308050, 5013066, 724799, 6139571, 4971226, 1993159, 1943079, 4204437, 4781256, 844028, 3942708, 4954544, 3133731, 1145509, 1367145, 2257681, 858584, 1067746, 4778518, 1231119, 4576045, 1297525, 1288400, 3682206, 1250131, 1182820, 5424259, 3709174, 2295689, 4061391, 5302154, 10559242, 7325562, 3387233, 7797666, 2753476, 2715522, 1173083, 1645081, 4592618, 4297214, 3949416, 5336682, 6877779, 3233152, 4060826, 7459817, 7298299, 3685784, 1900047, 4742040, 9953544, 3481398, 5678669, 4892205, 4397077, 2313034, 2668012, 8810424, 3604242, 2409661, 1394488, 831219, 4762689, 7039519, 2185957, 5008088, 1576152, 3085633, 4165506, 3684551, 2658662, 852356, 1116770, 1083987, 3935587, 3426586, 2915535, 1259030, 5380294, 1301253, 785464, 3880647, 4675712], "NumGC":18413} Allocated: 1591016288, active: 1592586240, metadata: 22460528 (n_thp 0), resident: 1729191936, mapped: 1744449536, retained: 4293251072 2021-09-13 18:46:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 246, "num_active_sctxs": 45, "num_free_sctxs": 201, "num_trim_sctxs": 210, "num_wctxs": 327, "num_free_wctxs": 310, "num_active_wctxs": 17, "num_trim_wctxs": 505, "num_readers": 104, "num_writers": 125, "cpu": 9.51552, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1760223232, "memory_jemalloc_allocated": 1642390096, "memory_in_use": 1751179264, "memory_stats_size_total": 1375288605, "memory_stats_size_global": 980771440, "memory_stats_size_page": 845029080, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 162568, "reclaim_pending": 57104900, "buf_memused": 337376553, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 88, "num_active_smr": 7, "num_pending_smr": 75, "num_pages": 2080, "items_count": 25, "total_records": 14878, "total_ops": 321157, "compacts": 1277868, "purges": 223, "lss_data_size": 417070435, "lss_used_space": 1983953607, "lss_fragmentation": 78, "lss_num_reads": 771948, "lss_read_bs": 119998058855, "lss_blk_read_bs": 118455865344, "rlss_num_reads": 23231, "lss_rdr_reads_bs": 7657924112, "lss_blk_rdr_reads_bs": 2843389952, "bytes_written": 108345724928, "bytes_incoming": 21051199036, "write_amp_avg": 2.03, "num_swapper_ctx": 3, "avg_sweep_interval": 161 s, "avg_run_interval": 99 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 10508, "resident_ratio": 70.62777, "cleaner_num_reads": 748345, "cleaner_read_bs": 111529759732, "cleaner_blk_read_bs": 114869796864, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 121248672, "recovery_used_space": 550553780, "recovery_lss_blk_read_bs": 84966633472, "recovery_lss_read_bs": 82555656714, "recovery_lss__num_reads": 635501, "recovery_bytes_written": 81923153920, "recovery_lss_frag": 77, "recovery_cleaner_num_reads": 635501, "recovery_cleaner_read_bs": 81868109791, "recovery_cleaner_blk_read_bs": 84280524800, "mvcc_purge_ratio": 595.12000 }2021-09-13 18:46:29 [ss-workload] instance inse% dele% read% 2021-09-13 18:46:29 [ss-workload] 0 14% 14% 70% 2021-09-13 18:46:29 [ss-workload] 1 14% 14% 70% 2021-09-13 18:46:31 [gsi-test] [] error: stats.ItemsCount < 0 2021-09-13 18:46:29 [ss-workload] 2 14% 14% 70% 2021-09-13 18:46:29 [ss-workload] 3 14% 14% 70% 2021-09-13 18:46:29 [ss-workload] 4 3% 3% 92% 2021-09-13 18:46:29 [ss-workload] 5 3% 3% 92% 2021-09-13 18:46:29 [ss-workload] 6 3% 3% 92% 2021-09-13 18:46:29 [ss-workload] 7 18% 18% 62% 2021-09-13 18:46:29 [ss-workload] 8 18% 18% 62% 2021-09-13 18:46:29 [ss-workload] 9 18% 18% 62% 2021-09-13 18:46:29 [ss-diag] instance num_rec_allocs num_rec_frees num_rec_swapout num_rec_swapin page_itemcnt page_cnt page_bytes num_pages memory_size memory_size_index cache_hit_ratio write_amp resident_ratio strike 2021-09-13 18:46:29 [ss-diag] 0 81586 80909 17084 17678 336 55 22024128 208 62560208 13577836 0.887097 6.296445 0.532651 0:0:1 2021-09-13 18:46:29 [ss-diag] 1 83182 83143 25754 26965 347 57 22745156 208 16663688 13577820 0.831111 8.141531 0.031200 0:0:0 2021-09-13 18:46:29 [ss-diag] 2 210430 209664 10116 11015 231 29 15141588 208 66075920 13577836 0.921182 0.877446 0.466785 0:0:0 2021-09-13 18:46:29 [ss-diag] 3 201056 200404 10928 11948 225 29 14748300 208 56849108 13577868 0.914754 0.877446 0.389952 0:0:0 2021-09-13 18:46:29 [ss-diag] 4 1415755 1414612 4340 4363 2601 463 170490348 208 88512344 13577820 0.999786 1.986011 0.980274 0:0:0 2021-09-13 18:46:29 [ss-diag] 5 3413749 3412197 4061 4097 4151 539 272089748 208 115323028 13577676 1.000000 0.877446 0.977330 0:0:0 2021-09-13 18:46:29 [ss-diag] 6 3428183 3426627 2579 2579 3777 491 247574796 208 115716644 13578028 0.999867 0.877446 1.000000 0:0:2 2021-09-13 18:46:29 [ss-diag] 7 142411 141446 16357 16610 408 70 26743584 208 84462544 13577628 0.921615 5.180702 0.792282 0:0:1 2021-09-13 18:46:29 [ss-diag] 8 370262 368748 9028 9234 360 45 23597280 208 113883836 13577708 0.954416 0.877446 0.880233 0:0:1 2021-09-13 18:46:29 [ss-diag] 9 380003 378600 9396 9644 783 100 51324084 208 106988465 13577852 0.944262 0.877446 0.851051 0:0:1 2021-09-13 18:46:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: starting... frag 89, data: 42274112, used: 395861524 log:(8786124882 - 9182642176) head:8786124882 2021-09-13 18:46:52 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : recoveryCleaner: completed... frag 86, data: 62534024, used: 458182164, disk: 461602734, relocated: 0, retries: 0, skipped: 5715356, cleaned:1153593590, log:(9939718472 - 10398162944), disk:(9936560210-10398162944), activeFragRatio:30, activeMaxFragRatio:80, run:289 duration:3141 ms elapsed: 299541 ms 2021-09-13T18:46:55.303-07:00 [Info] memstats {"Alloc":119108168, "TotalAlloc":1072721724552, "Sys":283252984, "Lookups":0, "Mallocs":90440411,"Frees":90419669, "HeapAlloc":119108168, "HeapSys":263258112, "HeapIdle":141287424, "HeapInuse":121970688,"HeapReleased":52838400, "HeapObjects":20742,"GCSys":8978432, "LastGC":1631584015279612967,"PauseTotalNs":51309145292, "PauseNs":[5943100, 2899779, 4909838, 2222087, 1396083, 7640667, 4432547, 8714874, 3888899, 2412831, 7950986, 4453206, 1125506, 2973704, 5389326, 858314, 5560708, 860935, 4736890, 2010762, 2158837, 3882203, 5932411, 3690191, 3021194, 5654978, 5693243, 5984360, 6883714, 1195403, 1835073, 943259, 3156206, 1754067, 1372966, 6496206, 4982677, 4099036, 1573962, 1557627, 1099369, 7858130, 7524163, 9096989, 2021182, 4667403, 2044872, 4185394, 7302910, 4306906, 1493916, 6289306, 1339319, 1784000, 1311721, 4196078, 1099078, 4061030, 3863675, 2718991, 1508991, 8416846, 6631226, 6001952, 9953960, 4565502, 9050199, 4754404, 1032872, 2819427, 7063246, 4324870, 4450972, 2437536, 2690462, 4150334, 895207, 3720658, 1621537, 2581122, 3616425, 4472922, 2629267, 5107878, 1743474, 5374898, 4062301, 7304679, 1865779, 6648057, 7515657, 4775020, 1409885, 5158585, 1779463, 4800179, 5257951, 8847197, 7893405, 6017054, 4913711, 1783038, 6616367, 4160258, 1265436, 2935453, 3066794, 2060794, 6848256, 4612647, 3198812, 1808474, 4469075, 7125693, 3750276, 6042899, 3013310, 8395315, 11027946, 4116634, 1052695, 6447991, 3897884, 3957717, 6449544, 2870160, 1594632, 6760078, 6724418, 3160357, 4071736, 1304829, 1737394, 2217082, 3852464, 1790158, 3871259, 5461389, 2636563, 1712000, 4503905, 6654946, 3414358, 4114915, 1160843, 2075947, 5041732, 6715850, 4847886, 1568175, 2597843, 900040, 3267994, 6921885, 5114881, 5239174, 6740410, 2891940, 4640114, 7695376, 4479123, 9610624, 1049306, 4090496, 1411522, 2815830, 487277, 1386323, 2956400, 7487544, 3981487, 4310926, 6044785, 4061031, 5193833, 1357910, 1360744, 1006623, 6559504, 5603970, 4645243, 2036892, 4239669, 4720376, 2119845, 5563456, 3497561, 4337531, 1986150, 5148325, 4615461, 4379469, 8856366, 2246835, 5598510, 4932192, 4080200, 8395523, 4241779, 1179346, 6480661, 8515549, 2305916, 10292507, 1909982, 3425051, 6402872, 5315615, 10092216, 7936727, 4732924, 5883635, 5605006, 7368485, 5676047, 4807787, 469501, 4934093, 7371964, 3926695, 9612639, 11411285, 1986958, 4063947, 4014739, 5791082, 7424601, 5450073, 4583007, 8597968, 5740342, 2200006, 4679827, 5356831, 2107552, 7433004, 1252520, 3536428, 3436494, 8202501, 2478202, 4635212, 5197903, 7150038, 7314986, 7727527, 1699946, 3913160, 7318394, 5385098, 8017038, 2938363, 7428642, 1603822, 1562545, 1724790], "NumGC":18973} Allocated: 1664717320, active: 1666289664, metadata: 22460528 (n_thp 0), resident: 1773002752, mapped: 1787604992, retained: 4250095616 2021-09-13 18:47:00 [gsi-test] [] { "memory_quota": 994832636, "count": 12670, "compacts": 12785, "purges": 0, "splits": 0, "merges": 0, "inserts": 12670, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 4, "memory_size": 63346496, "memory_size_index": 13577836, "allocated": 7117571228, "freed": 7054224732, "reclaimed": 7054093508, "reclaim_pending": 131224, "reclaim_list_size": 131224, "reclaim_list_count": 2, "reclaim_threshold": 7, "allocated_index": 13577836, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1278, "num_rec_allocs": 87730, "num_rec_frees": 87037, "num_rec_swapout": 20113, "num_rec_swapin": 19528, "num_rec_compressed": 0, "compresses": 24, "decompresses": 32, "num_compressed_pages": 0, "bytes_incoming": 830493160, "lss_data_size": 46575174, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 66535962, "est_recovery_size": 30957798, "lss_num_reads": 5077, "lss_read_bs": 1479113823, "lss_blk_read_bs": 944717824, "lss_rdr_reads_bs": 854484230, "lss_blk_rdr_reads_bs": 330153984, "cache_hits": 52319, "cache_misses": 1174, "cache_hit_ratio": 0.91317, "rlss_num_reads": 2844, "rcache_hits": 39713, "rcache_misses": 1110, "rcache_hit_ratio": 0.88525, "resident_ratio": 0.54225, "mvcc_purge_ratio": 0.00000, "currSn": 363231, "gcSn": 363230, "gcSnIntervals": "[0 181788 211314 240838 362915 363231]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 7, "num_free_wctxs": 23, "num_trim_wctxs": 77, "num_readers": 8, "num_writers": 9, "page_bytes": 17304672, "page_cnt": 43, "page_itemcnt": 264, "avg_item_size": 65548, "avg_page_size": 402434, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 141247893, "page_bytes_marshalled":13133124958, "page_bytes_compressed":7151792316, "compression_ratio": 1.83634, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 21696, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 23323504, "lss_data_size": 67433454, "lss_used_space": 97545744, "lss_disk_size": 166694912, "lss_fragmentation": 30, "lss_num_reads": 147576, "lss_read_bs": 21165498959, "lss_blk_read_bs": 21442801664, "bytes_written": 19439550464, "bytes_incoming": 830493160, "write_amp": 6.30, "write_amp_avg": 3.20, "lss_gc_num_reads": 144668, "lss_gc_reads_bs": 20207855921, "lss_blk_gc_reads_bs": 21023952896, "lss_gc_status": "frag 30, data: 46575174, used: 67129198, disk: 71778008, relocated: 1459240, retries: 862, skipped: 130426, cleaned:1075403893, log:(10341687442 - 10408816640), disk:(10337038632-10408816640), activeFragRatio:30, activeMaxFragRatio:80, run:179 duration:114927 ms elapsed: 273490 ms", "lss_head_offset": 10341687442, "lss_tail_offset": 10408816640, "num_sctxs": 32, "num_free_sctxs": 24, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:47:00 [gsi-test] [] { "memory_quota": 994832636, "count": 12362, "compacts": 12050, "purges": 0, "splits": 0, "merges": 0, "inserts": 12362, "deletes": 0, "compact_conflicts": 1, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 74, "persist_conflicts": 15, "memory_size": 53514448, "memory_size_index": 13577820, "allocated": 7363081392, "freed": 7309566944, "reclaimed": 7306747228, "reclaim_pending": 2819716, "reclaim_list_size": 787008, "reclaim_list_count": 5, "reclaim_threshold": 6, "allocated_index": 13577820, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": -1, "total_records": 1246, "num_rec_allocs": 88653, "num_rec_frees": 88140, "num_rec_swapout": 29415, "num_rec_swapin": 28682, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 810304376, "lss_data_size": 51474169, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 77991165, "est_recovery_size": 38301796, "lss_num_reads": 7500, "lss_read_bs": 2153262173, "lss_blk_read_bs": 1416073216, "lss_rdr_reads_bs": 1216266838, "lss_blk_rdr_reads_bs": 509542400, "cache_hits": 50164, "cache_misses": 1908, "cache_hit_ratio": 0.87912, "rlss_num_reads": 4115, "rcache_hits": 37949, "rcache_misses": 1761, "rcache_hit_ratio": 0.85581, "resident_ratio": 0.41172, "mvcc_purge_ratio": 0.00000, "currSn": 363231, "gcSn": 363227, "gcSnIntervals": "[0 182904 212424 241944 362914 363228]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 7, "num_free_wctxs": 22, "num_trim_wctxs": 52, "num_readers": 11, "num_writers": 11, "page_bytes": 27661256, "page_cnt": 70, "page_itemcnt": 422, "avg_item_size": 65548, "avg_page_size": 395160, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 147423785, "page_bytes_marshalled":13385232818, "page_bytes_compressed":7289613773, "compression_ratio": 1.83621, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 23280, "num_total_smr": 4, "num_active_smr": 6, "num_pending_smr": 2, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 35637659, "lss_data_size": 75282901, "lss_used_space": 114514072, "lss_disk_size": 168644608, "lss_fragmentation": 34, "lss_num_reads": 142552, "lss_read_bs": 21020058951, "lss_blk_read_bs": 21041065984, "bytes_written": 18649374720, "bytes_incoming": 810304376, "write_amp": 8.14, "write_amp_avg": 3.60, "lss_gc_num_reads": 138280, "lss_gc_reads_bs": 19682197056, "lss_blk_gc_reads_bs": 20445184000, "lss_gc_status": "frag 33, data: 52259196, used: 78509744 log:(11316383595 - 11395465216) head:11311726486", "lss_head_offset": 11325542735, "lss_tail_offset": 11402416128, "num_sctxs": 37, "num_free_sctxs": 24, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:47:02 [gsi-test] [] { "memory_quota": 994832636, "count": 11587, "compacts": 25737, "purges": 223, "splits": 30, "merges": 0, "inserts": 11587, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 1, "delete_conflicts": 0, "swapin_conflicts": 3, "persist_conflicts": 1, "memory_size": 78420128, "memory_size_index": 13577836, "allocated": 16105856891, "freed": 16027436763, "reclaimed": 16027436763, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577836, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 25, "total_records": 1710, "num_rec_allocs": 217214, "num_rec_frees": 216261, "num_rec_swapout": 12238, "num_rec_swapin": 11481, "num_rec_compressed": 0, "compresses": 32, "decompresses": 27, "num_compressed_pages": 0, "bytes_incoming": 759504676, "lss_data_size": 46713473, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 333667664, "est_recovery_size": 79946772, "lss_num_reads": 3598, "lss_read_bs": 1272036562, "lss_blk_read_bs": 551493632, "lss_rdr_reads_bs": 1075921830, "lss_blk_rdr_reads_bs": 366010368, "cache_hits": 48222, "cache_misses": 1027, "cache_hit_ratio": 0.89356, "rlss_num_reads": 3038, "rcache_hits": 36639, "rcache_misses": 1023, "rcache_hit_ratio": 0.86182, "resident_ratio": 0.55731, "mvcc_purge_ratio": 68.40000, "currSn": 362992, "gcSn": 362990, "gcSnIntervals": "[0 1626 2144 362735 362991]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 6, "num_free_wctxs": 27, "num_trim_wctxs": 70, "num_readers": 12, "num_writers": 10, "page_bytes": 29431052, "page_cnt": 54, "page_itemcnt": 449, "avg_item_size": 65548, "avg_page_size": 545019, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 159383236, "page_bytes_marshalled":6816406528, "page_bytes_compressed":3711087479, "compression_ratio": 1.83677, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 17096, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 219735714, "lss_data_size": 227927029, "lss_used_space": 1753263657, "lss_disk_size": 1810989056, "lss_fragmentation": 86, "lss_num_reads": 69170, "lss_read_bs": 18965743978, "lss_blk_read_bs": 15994609664, "bytes_written": 12540424192, "bytes_incoming": 12895585780, "write_amp": 0.88, "write_amp_avg": 0.71, "lss_gc_num_reads": 53965, "lss_gc_reads_bs": 13232659343, "lss_blk_gc_reads_bs": 13743075328, "lss_gc_status": "frag 86, data: 170403186, used: 1265251134 log:(50617697094 - 51883986944) head:50617697094", "lss_head_offset": 50683605381, "lss_tail_offset": 51964997632, "num_sctxs": 99, "num_free_sctxs": 86, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:47:03 [gsi-test] [] { "memory_quota": 994832636, "count": 11275, "compacts": 24783, "purges": 0, "splits": 0, "merges": 0, "inserts": 11275, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 16, "persist_conflicts": 8, "memory_size": 68256684, "memory_size_index": 13577868, "allocated": 15436601548, "freed": 15368344864, "reclaimed": 15348413232, "reclaim_pending": 19931632, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577868, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": -1, "total_records": 1691, "num_rec_allocs": 208062, "num_rec_frees": 207246, "num_rec_swapout": 13162, "num_rec_swapin": 12287, "num_rec_compressed": 0, "compresses": 13, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 739053700, "lss_data_size": 44317902, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 340906938, "est_recovery_size": 85979914, "lss_num_reads": 3744, "lss_read_bs": 1334055298, "lss_blk_read_bs": 560033792, "lss_rdr_reads_bs": 1175800841, "lss_blk_rdr_reads_bs": 401838080, "cache_hits": 46673, "cache_misses": 1118, "cache_hit_ratio": 0.93315, "rlss_num_reads": 3299, "rcache_hits": 35398, "rcache_misses": 1118, "rcache_hit_ratio": 0.91209, "resident_ratio": 0.48255, "mvcc_purge_ratio": 0.00000, "currSn": 362991, "gcSn": 362989, "gcSnIntervals": "[0 1615 2120 362734 362990]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 5, "num_free_wctxs": 24, "num_trim_wctxs": 58, "num_readers": 13, "num_writers": 10, "page_bytes": 26678036, "page_cnt": 53, "page_itemcnt": 407, "avg_item_size": 65548, "avg_page_size": 503359, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 147863387, "page_bytes_marshalled":7384125134, "page_bytes_compressed":4020319916, "compression_ratio": 1.83670, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 14688, "num_total_smr": 30, "num_active_smr": 3, "num_pending_smr": 26, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 219735714, "lss_data_size": 217907257, "lss_used_space": 1751826598, "lss_disk_size": 1817518080, "lss_fragmentation": 87, "lss_num_reads": 69269, "lss_read_bs": 18996329174, "lss_blk_read_bs": 16013029376, "bytes_written": 12546953216, "bytes_incoming": 12903582636, "write_amp": 0.88, "write_amp_avg": 0.71, "lss_gc_num_reads": 54004, "lss_gc_reads_bs": 13241206250, "lss_blk_gc_reads_bs": 13752995840, "lss_gc_status": "frag 86, data: 170403186, used: 1265251134 log:(50617697094 - 51883986944) head:50617697094", "lss_head_offset": 50687573943, "lss_tail_offset": 51969683456, "num_sctxs": 99, "num_free_sctxs": 76, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:47:03 [gsi-test] [] { "memory_quota": 994832636, "count": 89170, "compacts": 254298, "purges": 0, "splits": 0, "merges": 0, "inserts": 89170, "deletes": 0, "compact_conflicts": 5, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 9, "delete_conflicts": 0, "swapin_conflicts": 9, "persist_conflicts": 6, "memory_size": 88381184, "memory_size_index": 13577820, "allocated": 114596008752, "freed": 114507627568, "reclaimed": 114504742768, "reclaim_pending": 2884800, "reclaim_list_size": 2360304, "reclaim_list_count": 6, "reclaim_threshold": 1, "allocated_index": 13577820, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1176, "num_rec_allocs": 1493426, "num_rec_frees": 1492285, "num_rec_swapout": 4520, "num_rec_swapin": 4485, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 5844915160, "lss_data_size": 8543843, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 122054900, "est_recovery_size": 53055248, "lss_num_reads": 1874, "lss_read_bs": 552909773, "lss_blk_read_bs": 218320896, "lss_rdr_reads_bs": 513088489, "lss_blk_rdr_reads_bs": 178495488, "cache_hits": 1676875, "cache_misses": 594, "cache_hit_ratio": 1.00000, "rlss_num_reads": 1739, "rcache_hits": 1587706, "rcache_misses": 593, "rcache_hit_ratio": 1.00000, "resident_ratio": 0.97024, "mvcc_purge_ratio": 0.00000, "currSn": 415964, "gcSn": 415948, "gcSnIntervals": "[0 251866 281386 310907 410843 415949]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 7, "num_free_wctxs": 34, "num_trim_wctxs": 81, "num_readers": 16, "num_writers": 13, "page_bytes": 173505556, "page_cnt": 467, "page_itemcnt": 2647, "avg_item_size": 65548, "avg_page_size": 371532, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 105339459, "page_bytes_marshalled":61225586634, "page_bytes_compressed":33360090576, "compression_ratio": 1.83529, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 13352, "num_total_smr": 2, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 46466303, "lss_data_size": 12617191, "lss_used_space": 190402581, "lss_disk_size": 232767488, "lss_fragmentation": 93, "lss_num_reads": 256021, "lss_read_bs": 37524660915, "lss_blk_read_bs": 37778948096, "bytes_written": 36778483712, "bytes_incoming": 5844915160, "write_amp": 1.99, "write_amp_avg": 1.55, "lss_gc_num_reads": 254281, "lss_gc_reads_bs": 36898937063, "lss_blk_gc_reads_bs": 37487341568, "lss_gc_status": "frag 94, data: 7330560, used: 137422987 log:(36392792011 - 36530786304) head:36391531214", "lss_head_offset": 36520368681, "lss_tail_offset": 36658286592, "num_sctxs": 53, "num_free_sctxs": 45, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:47:03 [gsi-test] [] { "memory_quota": 994832636, "count": 68063, "compacts": 444340, "purges": 0, "splits": 0, "merges": 0, "inserts": 68063, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 24, "persist_conflicts": 36, "memory_size": 115323404, "memory_size_index": 13577676, "allocated": 258630007465, "freed": 258514684061, "reclaimed": 258490426133, "reclaim_pending": 24257928, "reclaim_list_size": 4130428, "reclaim_list_count": 7, "reclaim_threshold": 0, "allocated_index": 13577676, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": -1, "total_records": 1579, "num_rec_allocs": 3501271, "num_rec_frees": 3499719, "num_rec_swapout": 4591, "num_rec_swapin": 4564, "num_rec_compressed": 0, "compresses": 67, "decompresses": 201, "num_compressed_pages": 0, "bytes_incoming": 4461393524, "lss_data_size": 3330569, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 25619761, "est_recovery_size": 61585270, "lss_num_reads": 1744, "lss_read_bs": 606700670, "lss_blk_read_bs": 214298624, "lss_rdr_reads_bs": 604343855, "lss_blk_rdr_reads_bs": 211918848, "cache_hits": 1300867, "cache_misses": 606, "cache_hit_ratio": 0.99825, "rlss_num_reads": 1738, "rcache_hits": 1232804, "rcache_misses": 606, "rcache_hit_ratio": 0.99816, "resident_ratio": 0.98290, "mvcc_purge_ratio": 0.00000, "currSn": 411809, "gcSn": 411796, "gcSnIntervals": "[0 30229 40009 408442 411797]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 5, "num_free_wctxs": 24, "num_trim_wctxs": 70, "num_readers": 13, "num_writers": 16, "page_bytes": 280217700, "page_cnt": 542, "page_itemcnt": 4275, "avg_item_size": 65548, "avg_page_size": 517006, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 131417046, "page_bytes_marshalled":13493310688, "page_bytes_compressed":7351134237, "compression_ratio": 1.83554, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 12056, "num_total_smr": 36, "num_active_smr": 5, "num_pending_smr": 32, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 219735714, "lss_data_size": 216406477, "lss_used_space": 1752680604, "lss_disk_size": 1820516352, "lss_fragmentation": 87, "lss_num_reads": 69278, "lss_read_bs": 18999506987, "lss_blk_read_bs": 16015572992, "bytes_written": 12549951488, "bytes_incoming": 12906335652, "write_amp": 0.88, "write_amp_avg": 0.71, "lss_gc_num_reads": 54012, "lss_gc_reads_bs": 13243990796, "lss_blk_gc_reads_bs": 13755539456, "lss_gc_status": "frag 86, data: 170403186, used: 1265251134 log:(50617697094 - 51883986944) head:50617697094", "lss_head_offset": 50690110754, "lss_tail_offset": 51971760128, "num_sctxs": 99, "num_free_sctxs": 83, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:47:03 [gsi-test] [] { "memory_quota": 994832636, "count": 68490, "compacts": 445165, "purges": 0, "splits": 0, "merges": 0, "inserts": 68490, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 16, "persist_conflicts": 37, "memory_size": 113290784, "memory_size_index": 13578028, "allocated": 259121614004, "freed": 259008323220, "reclaimed": 259008323220, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13578028, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1548, "num_rec_allocs": 3508489, "num_rec_frees": 3506968, "num_rec_swapout": 3706, "num_rec_swapin": 3679, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 4489382520, "lss_data_size": 1185860, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 9122000, "est_recovery_size": 66175660, "lss_num_reads": 1381, "lss_read_bs": 480236073, "lss_blk_read_bs": 173125632, "lss_rdr_reads_bs": 472707158, "lss_blk_rdr_reads_bs": 164900864, "cache_hits": 1302747, "cache_misses": 471, "cache_hit_ratio": 0.99888, "rlss_num_reads": 1363, "rcache_hits": 1234257, "rcache_misses": 471, "rcache_hit_ratio": 0.99882, "resident_ratio": 0.98256, "mvcc_purge_ratio": 0.00000, "currSn": 411583, "gcSn": 411567, "gcSnIntervals": "[0 29933 39667 408214 411568]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 4, "num_free_wctxs": 34, "num_trim_wctxs": 71, "num_readers": 15, "num_writers": 16, "page_bytes": 212703260, "page_cnt": 430, "page_itemcnt": 3245, "avg_item_size": 65548, "avg_page_size": 494658, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 131130432, "page_bytes_marshalled":12267542616, "page_bytes_compressed":6683313909, "compression_ratio": 1.83555, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 11672, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 219735714, "lss_data_size": 216406477, "lss_used_space": 1752680604, "lss_disk_size": 1820516352, "lss_fragmentation": 87, "lss_num_reads": 69278, "lss_read_bs": 18999506987, "lss_blk_read_bs": 16015572992, "bytes_written": 12549951488, "bytes_incoming": 12906335652, "write_amp": 0.88, "write_amp_avg": 0.71, "lss_gc_num_reads": 54012, "lss_gc_reads_bs": 13243990796, "lss_blk_gc_reads_bs": 13755539456, "lss_gc_status": "frag 86, data: 170403186, used: 1265251134 log:(50617697094 - 51883986944) head:50617697094", "lss_head_offset": 50690110754, "lss_tail_offset": 51971760128, "num_sctxs": 99, "num_free_sctxs": 83, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:47:04 [gsi-test] [] { "memory_quota": 994832636, "count": 21336, "compacts": 23537, "purges": 0, "splits": 0, "merges": 0, "inserts": 21336, "deletes": 0, "compact_conflicts": 1, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 3, "persist_conflicts": 3, "memory_size": 65577608, "memory_size_index": 13577628, "allocated": 11936108628, "freed": 11870531020, "reclaimed": 11870531020, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 2, "allocated_index": 13577628, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1196, "num_rec_allocs": 151086, "num_rec_frees": 150383, "num_rec_swapout": 18709, "num_rec_swapin": 18216, "num_rec_compressed": 0, "compresses": 14, "decompresses": 42, "num_compressed_pages": 0, "bytes_incoming": 1398532128, "lss_data_size": 43328709, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 63718689, "est_recovery_size": 31613554, "lss_num_reads": 4942, "lss_read_bs": 1433228609, "lss_blk_read_bs": 892702720, "lss_rdr_reads_bs": 911556874, "lss_blk_rdr_reads_bs": 378675200, "cache_hits": 58747, "cache_misses": 1340, "cache_hit_ratio": 0.93603, "rlss_num_reads": 3080, "rcache_hits": 37460, "rcache_misses": 1291, "rcache_hit_ratio": 0.90547, "resident_ratio": 0.58779, "mvcc_purge_ratio": 0.00000, "currSn": 363754, "gcSn": 363752, "gcSnIntervals": "[0 182031 211556 241076 363398 363753]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 7, "num_free_wctxs": 16, "num_trim_wctxs": 55, "num_readers": 3, "num_writers": 11, "page_bytes": 30152080, "page_cnt": 78, "page_itemcnt": 460, "avg_item_size": 65548, "avg_page_size": 386565, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 132890274, "page_bytes_marshalled":23729269340, "page_bytes_compressed":12928613349, "compression_ratio": 1.83541, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 24248, "num_total_smr": 1, "num_active_smr": 4, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 34917344, "lss_data_size": 63793593, "lss_used_space": 93869333, "lss_disk_size": 154476544, "lss_fragmentation": 32, "lss_num_reads": 187333, "lss_read_bs": 26947429414, "lss_blk_read_bs": 27182407680, "bytes_written": 25249632256, "bytes_incoming": 1398532128, "write_amp": 5.18, "write_amp_avg": 2.93, "lss_gc_num_reads": 184202, "lss_gc_reads_bs": 25925546983, "lss_blk_gc_reads_bs": 26704322560, "lss_gc_status": "frag 35, data: 39973403, used: 62001192 log:(13043824179 - 13106077696) head:13043824179", "lss_head_offset": 13053024649, "lss_tail_offset": 13116715008, "num_sctxs": 29, "num_free_sctxs": 18, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:47:04 [gsi-test] [] { "memory_quota": 994832636, "count": 18549, "compacts": 45573, "purges": 0, "splits": 0, "merges": 0, "inserts": 18549, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 8, "delete_conflicts": 0, "swapin_conflicts": 1, "persist_conflicts": 2, "memory_size": 105230756, "memory_size_index": 13577708, "allocated": 28188496356, "freed": 28083265600, "reclaimed": 28055335792, "reclaim_pending": 27929808, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577708, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": -1, "total_records": 1729, "num_rec_allocs": 382292, "num_rec_frees": 380918, "num_rec_swapout": 10604, "num_rec_swapin": 10249, "num_rec_compressed": 0, "compresses": 25, "decompresses": 49, "num_compressed_pages": 3, "bytes_incoming": 1215849852, "lss_data_size": 27613430, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 212411000, "est_recovery_size": 86766896, "lss_num_reads": 3102, "lss_read_bs": 1142831434, "lss_blk_read_bs": 479817728, "lss_rdr_reads_bs": 1051501540, "lss_blk_rdr_reads_bs": 388554752, "cache_hits": 51419, "cache_misses": 1062, "cache_hit_ratio": 0.87705, "rlss_num_reads": 2901, "rcache_hits": 32870, "rcache_misses": 1062, "rcache_hit_ratio": 0.82353, "resident_ratio": 0.79468, "mvcc_purge_ratio": 0.00000, "currSn": 363402, "gcSn": 363400, "gcSnIntervals": "[0 1832 2414 363105 363401]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 7, "num_free_wctxs": 22, "num_trim_wctxs": 33, "num_readers": 6, "num_writers": 12, "page_bytes": 31987424, "page_cnt": 60, "page_itemcnt": 488, "avg_item_size": 65548, "avg_page_size": 533123, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 143573412, "page_bytes_marshalled":8053184440, "page_bytes_compressed":4384878466, "compression_ratio": 1.83658, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 15224, "num_total_smr": 38, "num_active_smr": 1, "num_pending_smr": 36, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 219735714, "lss_data_size": 217760785, "lss_used_space": 1771758068, "lss_disk_size": 1864835072, "lss_fragmentation": 87, "lss_num_reads": 69507, "lss_read_bs": 19069217698, "lss_blk_read_bs": 16058429440, "bytes_written": 12594270208, "bytes_incoming": 12929343000, "write_amp": 0.88, "write_amp_avg": 0.71, "lss_gc_num_reads": 54126, "lss_gc_reads_bs": 13272040699, "lss_blk_gc_reads_bs": 13784174592, "lss_gc_status": "frag 86, data: 170403186, used: 1265251134 log:(50617697094 - 51883986944) head:50617697094", "lss_head_offset": 50709216452, "lss_tail_offset": 52004032512, "num_sctxs": 99, "num_free_sctxs": 85, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:47:05 [gsi-test] [] { "memory_quota": 994832636, "count": 19010, "compacts": 46613, "purges": 0, "splits": 0, "merges": 0, "inserts": 19010, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 3, "delete_conflicts": 0, "swapin_conflicts": 1, "persist_conflicts": 2, "memory_size": 93560868, "memory_size_index": 13577852, "allocated": 28801443292, "freed": 28707882424, "reclaimed": 28696474288, "reclaim_pending": 11408136, "reclaim_list_size": 1376804, "reclaim_list_count": 2, "reclaim_threshold": 0, "allocated_index": 13577852, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1703, "num_rec_allocs": 390595, "num_rec_frees": 389397, "num_rec_swapout": 11040, "num_rec_swapin": 10535, "num_rec_compressed": 0, "compresses": 9, "decompresses": 12, "num_compressed_pages": 0, "bytes_incoming": 1246067480, "lss_data_size": 28468492, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 218988400, "est_recovery_size": 83815902, "lss_num_reads": 3108, "lss_read_bs": 1117593904, "lss_blk_read_bs": 476356608, "lss_rdr_reads_bs": 1017500944, "lss_blk_rdr_reads_bs": 375468032, "cache_hits": 52593, "cache_misses": 1038, "cache_hit_ratio": 0.93277, "rlss_num_reads": 2831, "rcache_hits": 33583, "rcache_misses": 1038, "rcache_hit_ratio": 0.89744, "resident_ratio": 0.70346, "mvcc_purge_ratio": 0.00000, "currSn": 363483, "gcSn": 363479, "gcSnIntervals": "[0 1854 2478 363187 363480]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 11, "num_free_wctxs": 18, "num_trim_wctxs": 37, "num_readers": 5, "num_writers": 13, "page_bytes": 40574212, "page_cnt": 77, "page_itemcnt": 619, "avg_item_size": 65548, "avg_page_size": 526937, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 140224165, "page_bytes_marshalled":8733452800, "page_bytes_compressed":4757932577, "compression_ratio": 1.83556, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 14288, "num_total_smr": 13, "num_active_smr": 10, "num_pending_smr": 7, "lss_stats": { "shared": true, "punch_hole_support": true, "buf_memused": 219735714, "lss_data_size": 212736806, "lss_used_space": 1772430796, "lss_disk_size": 1870229504, "lss_fragmentation": 87, "lss_num_reads": 69565, "lss_read_bs": 19089716826, "lss_blk_read_bs": 16069701632, "bytes_written": 12599664640, "bytes_incoming": 12935111224, "write_amp": 0.88, "write_amp_avg": 0.71, "lss_gc_num_reads": 54141, "lss_gc_reads_bs": 13277075787, "lss_blk_gc_reads_bs": 13789290496, "lss_gc_status": "frag 86, data: 170403186, used: 1265251134 log:(50617697094 - 51883986944) head:50617697094", "lss_head_offset": 50713293977, "lss_tail_offset": 52008505344, "num_sctxs": 99, "num_free_sctxs": 81, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:47:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 56, data: 13711568, used: 31731642 log:(14993486314 - 15025352704) head:14993486314 2021-09-13 18:47:05 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 30, data: 22694128, used: 32653242, disk: 35024616, relocated: 29057693, retries: 9286, skipped: 2397101, cleaned:3183103930, log:(18176590244 - 18209509376), disk:(18174484760-18209509376), activeFragRatio:30, activeMaxFragRatio:80, run:266 duration:80545 ms elapsed: 288007 ms 2021-09-13 18:47:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoveryCleaner: starting... frag 43, data: 23743184, used: 42135272 log:(17526223128 - 17569280000) head:17526223128 2021-09-13 18:47:05 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : recoveryCleaner: completed... frag 30, data: 23612052, used: 33971746, disk: 39501334, relocated: 26645131, retries: 12736, skipped: 2429858, cleaned:2389508224, log:(19915731352 - 19949834240), disk:(19910332906-19949834240), activeFragRatio:30, activeMaxFragRatio:80, run:259 duration:52174 ms elapsed: 288094 ms 2021-09-13 18:47:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 91, data: 2630896, used: 31338006 log:(77464953322 - 77496291328) head:77463900580 2021-09-13 18:47:05 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 92, data: 3548820, used: 49769984, disk: 50822818, relocated: 36462638, retries: 133656, skipped: 8112756, cleaned:9466183610, log:(86931136932 - 86980907008), disk:(86930084190-86980907008), activeFragRatio:30, activeMaxFragRatio:80, run:258 duration:227419 ms elapsed: 300708 ms 2021-09-13 18:47:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: starting... frag 66, data: 9449760, used: 28503924 log:(66757964266 - 66786734080) head:66756911524 2021-09-13 18:47:06 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : recoveryCleaner: completed... frag 37, data: 18891300, used: 30350870, disk: 31665896, relocated: 33645131, retries: 20046, skipped: 3694769, cleaned:6191779700, log:(72949743966 - 72980357120), disk:(72948691224-72980357120), activeFragRatio:30, activeMaxFragRatio:80, run:267 duration:148576 ms elapsed: 288850 ms 2021-09-13 18:47:17 [gsi-test] [] Stop and verify... 2021-09-13 18:47:18 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : Swapper stopped 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_0_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex(shard1) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_7_0/docIndex/recovery(shard1) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:47:18 [plasma] [i] Shard /data/gsi_testdir/shards/shard1(1) : All daemons stopped 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_4_0/docIndex/recovery(shard1) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:47:18 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : Swapper stopped 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex/recovery(shard2) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:47:18 [plasma] [i] Shard /data/gsi_testdir/shards/shard2(2) : All daemons stopped 2021-09-13 18:47:18 [plasma] [i] LSS /data/gsi_testdir/devel_1_0/docIndex(shard2) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:47:19 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : Swapper stopped 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Begin stopping data LSS Cleaner 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.Stop: Done stopping data LSS Cleaner 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: Begin stopping daemons 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Begin stopping recovery LSS Cleaner 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Received stop message 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoveryCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : RecoverCleaner.Stop: Done stopping recovery LSS Cleaner 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Received stop message 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.runtimeStats: Stopped daemon 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data/recovery(shard3) : LSSCtx.stopDaemons: all deamons stopped 2021-09-13 18:47:19 [plasma] [i] Shard /data/gsi_testdir/shards/shard3(3) : All daemons stopped 2021-09-13 18:47:19 [gsi-test] [0] Stop and verify... 2021-09-13 18:47:19 [plasma] [i] LSS /data/gsi_testdir/shards/shard3/data(shard3) : LSSCleaner.lssCleanerDaemon: Stopped daemon 2021-09-13 18:47:20 [gsi-test] [0] Stop and verify... OK 2021-09-13 18:47:20 [gsi-test] [1] Stop and verify... 2021-09-13 18:47:22 [gsi-test] [1] Stop and verify... OK 2021-09-13 18:47:22 [gsi-test] [2] Stop and verify... Mon Sep 13 18:47:23 PDT 2021 RSS = 1625736 2021-09-13T18:47:25.560-07:00 [Info] memstats {"Alloc":83270544, "TotalAlloc":1097621272808, "Sys":283252984, "Lookups":0, "Mallocs":92744428,"Frees":92728101, "HeapAlloc":83270544, "HeapSys":263946240, "HeapIdle":177684480, "HeapInuse":86261760,"HeapReleased":133267456, "HeapObjects":16327,"GCSys":8978432, "LastGC":1631584045553606759,"PauseTotalNs":52986307662, "PauseNs":[5144619, 728899, 1981100, 5868497, 2987356, 1609720, 1772329, 1121655, 6338503, 1409353, 3198703, 2229637, 1339213, 4052144, 8232793, 4563872, 5635344, 1768651, 4001492, 2452518, 6616119, 8020884, 5041392, 5776337, 1378729, 5577841, 2790616, 2186572, 5363470, 1223915, 4695474, 6153158, 4015411, 2706061, 988409, 3034680, 1182334, 1233730, 4975424, 767920, 2963859, 4851630, 6502917, 2575689, 3372622, 6475839, 4345023, 4458770, 3615521, 873316, 3129374, 5260241, 7873744, 3516457, 5302441, 4068775, 3862378, 4593364, 5187102, 3996642, 7390083, 6360781, 1882068, 846141, 5041674, 4201154, 7447592, 4977477, 1979076, 4677870, 4065466, 5989254, 4173445, 1312172, 4247481, 4157550, 5251668, 2491160, 3505246, 7954898, 4607173, 1602755, 4792025, 2421188, 3881240, 7645997, 6242721, 5842325, 969876, 1689142, 3994373, 2654515, 3582836, 2721236, 5379730, 3767584, 1056293, 2532160, 6317545, 2726214, 1482987, 7979785, 2037120, 1599284, 4596184, 4299751, 3395078, 1267410, 6414006, 5921306, 4201221, 6384152, 2603240, 9785172, 5050101, 1287457, 3529106, 4476055, 6476942, 1256355, 5628003, 4626196, 6467674, 2554615, 710717, 5156409, 1039279, 2615721, 7747642, 2301972, 1881995, 414659, 6736571, 8836271, 5143755, 1094146, 3292610, 1435551, 3627533, 5532814, 6369727, 6795734, 1551720, 3983707, 4588322, 6832863, 438550, 4249697, 661929, 2776760, 2263053, 5103823, 2641193, 8305189, 1113166, 3833056, 5810144, 3252673, 7346020, 3964979, 3974043, 1227006, 3593404, 4413475, 1465228, 3373413, 4605349, 2518394, 8402444, 6118598, 2689550, 5559854, 3979931, 4125066, 2289938, 7366298, 2514388, 4264794, 4852822, 6934135, 6561427, 3809682, 4541520, 3229274, 4090062, 5858180, 822355, 1315528, 3496722, 601386, 944327, 446920, 836836, 894878, 388520, 1449048, 393159, 3764508, 513356, 356137, 716919, 514596, 1347078, 523170, 381398, 476124, 703661, 4491518, 820452, 531833, 1560480, 417377, 350071, 491799, 883552, 465351, 391844, 770748, 4150341, 965136, 1499470, 444391, 577567, 367617, 502596, 3175095, 372313, 339033, 956315, 324146, 774435, 368332, 791499, 547328, 1029209, 3380563, 2780523, 4688752, 3001590, 7723833, 10679322, 3558119, 1143602, 5136423, 6505769, 1523776, 3121811, 8350661, 7636457, 5273220, 4452137, 3436330, 1081602, 766816, 1856460, 1876978], "NumGC":19435} Allocated: 1638596144, active: 1640194048, metadata: 22460528 (n_thp 0), resident: 1699631104, mapped: 1714692096, retained: 4323008512 2021-09-13 18:47:42 [ss-workload] instance inse% dele% read% 2021-09-13 18:47:42 [ss-workload] 0 14% 14% 70% 2021-09-13 18:47:42 [ss-workload] 1 14% 14% 70% 2021-09-13T18:47:55.603-07:00 [Info] memstats {"Alloc":80364648, "TotalAlloc":1107059309808, "Sys":283252984, "Lookups":0, "Mallocs":93421830,"Frees":93404883, "HeapAlloc":80364648, "HeapSys":263946240, "HeapIdle":180731904, "HeapInuse":83214336,"HeapReleased":155762688, "HeapObjects":16947,"GCSys":8978432, "LastGC":1631584075602332162,"PauseTotalNs":53189421558, "PauseNs":[417961, 837045, 342875, 1206549, 7669510, 383752, 1420456, 734448, 1105714, 1294768, 539007, 566919, 444079, 449534, 446872, 1010002, 367451, 1070909, 547478, 411968, 2596628, 729513, 958038, 370166, 1378806, 1127092, 2349266, 376398, 1054291, 2707364, 3014662, 514642, 375272, 670667, 467240, 729027, 440426, 412640, 2356576, 420652, 432520, 771338, 469067, 987317, 606960, 3264256, 379625, 535672, 391156, 363844, 856748, 1076589, 793888, 557151, 335946, 1491383, 445104, 505220, 572818, 418559, 680985, 1030194, 1146268, 838843, 492299, 651429, 1662454, 1188303, 417601, 1633267, 412781, 1679932, 589417, 306170, 767986, 314769, 717938, 453800, 689211, 5060495, 395301, 560068, 406836, 406855, 473246, 570159, 1710589, 499034, 520144, 746784, 352971, 984072, 453509, 867193, 610682, 2742036, 1071026, 553175, 1244974, 1154220, 825524, 545045, 6072386, 864375, 857174, 1488954, 484904, 748300, 430090, 1298842, 389756, 395495, 461778, 445311, 1065661, 1936195, 5808754, 397175, 396173, 852286, 1082629, 388636, 925626, 516386, 1192657, 780623, 410640, 406860, 1049044, 399936, 490094, 373014, 1191729, 346792, 495526, 626743, 2108230, 1209484, 1745590, 773120, 4159664, 459103, 674641, 404478, 492575, 473036, 979718, 516735, 486698, 417218, 978479, 419836, 509307, 908217, 2276816, 992271, 1604591, 872184, 465534, 1420806, 504874, 487566, 1095909, 402522, 1120662, 366561, 888606, 585350, 426727, 1214724, 964064, 732356, 490660, 1354897, 3150370, 509509, 409591, 386409, 1023138, 555071, 424924, 382197, 350331, 949764, 4573486, 378884, 742098, 528069, 951907, 439131, 356381, 452392, 332062, 686742, 906071, 1149339, 456471, 368274, 400769, 569249, 994786, 1465559, 1808237, 437654, 602378, 437003, 1172621, 1176891, 1159867, 449313, 1331795, 473274, 320992, 320203, 476932], "NumGC":19650} Allocated: 1691839984, active: 1693446144, metadata: 22460528 (n_thp 0), resident: 1758928896, mapped: 1773531136, retained: 4264169472 Mon Sep 13 18:48:23 PDT 2021 RSS = 1687000 2021-09-13T18:48:25.646-07:00 [Info] memstats {"Alloc":75696824, "TotalAlloc":1116102139176, "Sys":283252984, "Lookups":0, "Mallocs":94070285,"Frees":94052455, "HeapAlloc":75696824, "HeapSys":263946240, "HeapIdle":185442304, "HeapInuse":78503936,"HeapReleased":161726464, "HeapObjects":17830,"GCSys":8978432, "LastGC":1631584105592646834,"PauseTotalNs":53371187468, "PauseNs":[453227, 1013589, 1006168, 446113, 318255, 827538, 523113, 372113, 801480, 491099, 897033, 1411149, 370432, 1355324, 411540, 602102, 417990, 1158392, 739293, 349890, 466429, 9164125, 385243, 384731, 529428, 749762, 1076374, 451748, 1037545, 422247, 729321, 1012372, 3552564, 512628, 1012336, 1375382, 759811, 1887599, 566368, 1038836, 475096, 526317, 378710, 343361, 851194, 551221, 418362, 482397, 816727, 767683, 737218, 919843, 600138, 523385, 675552, 1019698, 571543, 437255, 365750, 518272, 468000, 403723, 327066, 1153521, 394891, 940933, 416284, 473837, 1301860, 789674, 1173249, 1098493, 461848, 503092, 485749, 1208155, 496076, 1719540, 353196, 625059, 1902676, 727509, 389304, 885431, 5907699, 354701, 356317, 518063, 2040425, 373278, 1014898, 1076412, 384931, 782267, 405448, 898415, 463534, 383944, 686062, 355747, 534236, 434302, 517792, 1319277, 376126, 1613601, 576610, 595188, 903030, 1272949, 940141, 505098, 554492, 556741, 376042, 1508013, 1439548, 489110, 1131154, 381569, 1125873, 447189, 615593, 1104128, 1345768, 1192238, 428237, 834338, 697108, 363036, 996830, 985345, 361592, 417990, 416121, 531812, 407314, 1697219, 1566181, 1649787, 410518, 2357281, 433201, 350631, 897706, 498763, 372212, 478258, 482903, 1345614, 403451, 614586, 368127, 685604, 588624, 770683, 457265, 944699, 852020, 397439, 359807, 477015, 737951, 481739, 537612, 1137862, 492833, 418223, 1041854, 456310, 517285, 943312, 458000, 566384, 1893348, 574679, 1681251, 2078480, 412447, 334070, 648891, 798243, 2426263, 443234, 1511752, 338925, 363624, 729124, 1182651, 4431034, 509772, 542507, 448798, 797105, 502622, 343902, 1277449, 6254771, 875813, 2874518, 950237, 354421, 635487, 565318, 337020, 426776, 388415, 526755], "NumGC":19858} Allocated: 1712959032, active: 1714573312, metadata: 22460528 (n_thp 0), resident: 1775566848, mapped: 1790169088, retained: 4247531520 2021-09-13T18:48:55.692-07:00 [Info] memstats {"Alloc":69471232, "TotalAlloc":1125064274984, "Sys":283252984, "Lookups":0, "Mallocs":94713902,"Frees":94696748, "HeapAlloc":69471232, "HeapSys":264011776, "HeapIdle":191668224, "HeapInuse":72343552,"HeapReleased":155328512, "HeapObjects":17154,"GCSys":8912896, "LastGC":1631584135571118085,"PauseTotalNs":53553396682, "PauseNs":[822810, 611763, 406060, 1383361, 619349, 989990, 1260079, 1049557, 519326, 380263, 5414523, 548978, 557497, 548328, 529224, 1108714, 955879, 1177013, 961128, 362051, 2255294, 404509, 435655, 559377, 642499, 425389, 402169, 545028, 387200, 348740, 1007248, 784776, 441550, 967967, 843297, 406264, 365699, 332159, 684181, 1116955, 411808, 580121, 881641, 969681, 387855, 1297474, 953360, 1042717, 483259, 375739, 800823, 447433, 466210, 1870914, 1327168, 923137, 914586, 568233, 926638, 853908, 843413, 421219, 503508, 1871389, 379757, 306453, 600401, 978854, 1841782, 4546523, 437262, 833899, 804328, 562197, 425003, 476495, 1017053, 386762, 377916, 1833844, 457128, 1082702, 404108, 375995, 5403957, 509800, 1145170, 318980, 3713613, 1072285, 358355, 1104109, 2066985, 443540, 1086079, 442501, 1495197, 1383572, 856051, 771126, 355693, 1393127, 2343958, 806995, 1461213, 521911, 405961, 447372, 522008, 385458, 493591, 395537, 904706, 559540, 381208, 614875, 487964, 570660, 381755, 978327, 1373444, 491789, 2334802, 402668, 707777, 1004840, 1106608, 467611, 758375, 420028, 1141517, 400893, 1071154, 437086, 2948320, 546094, 453263, 778447, 1267715, 750237, 584964, 408024, 411887, 1456070, 541506, 450259, 1119364, 1570559, 698372, 1030565, 457000, 474120, 836996, 947343, 501211, 763200, 579619, 1280993, 500796, 588522, 2659983, 518858, 744885, 405294, 453828, 429477, 784871, 910796, 456325, 910553, 903585, 370800, 473775, 493995, 558978, 868167, 413623, 1833804, 475890, 478836, 669001, 427311, 428357, 430128, 1005153, 484498, 385481, 682700, 520053, 1206860, 426528, 554483, 432761, 771283, 454305, 799503, 729701, 411095, 392456, 1353029, 963044, 749013, 680259, 5956416, 599819, 735471, 1497659, 453733, 403029], "NumGC":20067} Allocated: 1717792312, active: 1719390208, metadata: 22460528 (n_thp 0), resident: 1769340928, mapped: 1783943168, retained: 4253757440 Mon Sep 13 18:49:23 PDT 2021 RSS = 1700116 2021-09-13T18:49:25.727-07:00 [Info] memstats {"Alloc":62246120, "TotalAlloc":1134409203888, "Sys":283252984, "Lookups":0, "Mallocs":95370483,"Frees":95353404, "HeapAlloc":62246120, "HeapSys":263913472, "HeapIdle":198762496, "HeapInuse":65150976,"HeapReleased":158425088, "HeapObjects":17079,"GCSys":9011200, "LastGC":1631584165708107437,"PauseTotalNs":53712715371, "PauseNs":[579264, 485264, 413398, 443381, 1051411, 1142025, 433240, 765166, 993741, 502625, 385674, 787733, 581841, 1046174, 2187769, 360535, 479280, 1168430, 383926, 1005250, 1219196, 903770, 526531, 674730, 1045446, 433870, 455820, 553994, 516222, 381751, 1068083, 864179, 1005365, 628299, 467009, 843338, 445890, 570652, 482614, 594203, 949257, 372995, 367658, 576698, 839204, 507452, 417881, 1018524, 1538607, 497670, 537058, 943112, 427316, 383192, 414864, 283981, 364647, 469620, 1564713, 437236, 1035126, 1246562, 1277029, 1322754, 410506, 1245886, 802320, 1062550, 1585347, 502562, 575501, 417976, 545345, 528193, 449344, 360833, 913904, 339586, 1180424, 767830, 1016398, 1045971, 442787, 2000379, 1211434, 946367, 990303, 1139943, 1144110, 522637, 668319, 566140, 768110, 1707392, 386765, 406895, 550470, 1015450, 391975, 448537, 429924, 473148, 1130475, 943243, 594156, 324817, 710913, 514976, 387720, 1017466, 1333641, 423185, 1056787, 433537, 657895, 859214, 1041146, 697138, 484370, 555138, 430854, 1041193, 694317, 1176331, 595350, 1237532, 935886, 409481, 360333, 422573, 356568, 838021, 511718, 375515, 925178, 363169, 733333, 386539, 476306, 482264, 1083669, 523095, 414395, 834547, 793580, 1209773, 424697, 410033, 327376, 845238, 671931, 1176220, 469152, 477516, 410338, 442788, 541666, 392403, 829218, 507539, 4275221, 458357, 1264515, 725784, 1121807, 451174, 769365, 568914, 431677, 479549, 392515, 440546, 1025724, 570709, 1408942, 1189110, 524183, 422147, 1350380, 1004747, 1487125, 501840, 1801221, 422766, 711579, 746033, 1373841, 404066, 505245, 392644, 874427, 549141, 630256, 499149, 389443, 397897, 455506, 457316, 615615, 952882, 465233, 477690, 384036, 456267, 522493, 614586, 548664, 1039393, 783279, 854045, 913788, 1145261, 1152845, 452285, 362796, 767412], "NumGC":20283} Allocated: 1719089448, active: 1720705024, metadata: 22460528 (n_thp 0), resident: 1794330624, mapped: 1808932864, retained: 4228767744 2021-09-13T18:49:55.761-07:00 [Info] memstats {"Alloc":72645520, "TotalAlloc":1143461518520, "Sys":283252984, "Lookups":0, "Mallocs":96017315,"Frees":96001131, "HeapAlloc":72645520, "HeapSys":263913472, "HeapIdle":188334080, "HeapInuse":75579392,"HeapReleased":164569088, "HeapObjects":16184,"GCSys":9011200, "LastGC":1631584195712399038,"PauseTotalNs":53878033258, "PauseNs":[420134, 676553, 495396, 607519, 333630, 425627, 751446, 1408371, 840521, 1041521, 539982, 402771, 1117589, 1155291, 459550, 522815, 848368, 371356, 397647, 820486, 1183793, 1211148, 1646023, 435643, 1341969, 572261, 1512420, 484823, 480629, 417740, 394195, 411418, 968619, 508001, 2653161, 406088, 1076062, 938424, 1116378, 451987, 416139, 354916, 426202, 1224329, 1704012, 521134, 1076109, 8519400, 365095, 387499, 1211133, 1484422, 414258, 999851, 423210, 347805, 439524, 487749, 385115, 565238, 1283895, 992026, 405232, 1131398, 443237, 615780, 1333286, 451542, 435694, 400769, 359835, 586448, 392318, 665646, 1326960, 1159340, 481852, 518424, 424403, 1407833, 608205, 1083549, 1033854, 1630678, 360266, 532621, 1817542, 1659606, 341798, 846633, 654827, 735602, 397414, 554118, 390593, 1337602, 382159, 977024, 533500, 517505, 888530, 378068, 490071, 968229, 399247, 1480467, 558590, 826709, 795811, 1110469, 441635, 395711, 627420, 1549753, 420382, 952096, 546128, 414726, 1558880, 430051, 1045784, 374107, 685872, 385570, 1001434, 619478, 1283553, 470334, 704400, 901136, 1125343, 1099220, 357580, 419215, 1366020, 448960, 508518, 1061195, 450433, 527128, 529156, 434110, 1025484, 588967, 729277, 566051, 345199, 460716, 1380932, 1321208, 905274, 403580, 1338537, 438200, 1183502, 526672, 1928308, 1050435, 356473, 354262, 546714, 517441, 336629, 427416, 453693, 517996, 1009465, 931490, 1242532, 1553550, 594311, 898734, 491100, 526923, 1014504, 469817, 503859, 467055, 407661, 376962, 441954, 840320, 410350, 326090, 411403, 1079775, 798275, 758336, 1178937, 398392, 441360, 668215, 952638, 800573, 398615, 540425, 1031402, 528344, 961928, 677883, 425331, 517908, 1988661, 347211, 738971, 979377, 454554, 1733262, 453440], "NumGC":20492} Allocated: 1725483120, active: 1727107072, metadata: 22460528 (n_thp 0), resident: 1786626048, mapped: 1801228288, retained: 4236472320 Mon Sep 13 18:50:23 PDT 2021 RSS = 1713820 2021-09-13T18:50:25.815-07:00 [Info] memstats {"Alloc":83093704, "TotalAlloc":1152652488952, "Sys":283252984, "Lookups":0, "Mallocs":96671451,"Frees":96653544, "HeapAlloc":83093704, "HeapSys":263946240, "HeapIdle":177979392, "HeapInuse":85966848,"HeapReleased":156942336, "HeapObjects":17907,"GCSys":8978432, "LastGC":1631584225777605898,"PauseTotalNs":54063669633, "PauseNs":[1458218, 706580, 704509, 477584, 372238, 1285947, 1824300, 923057, 843591, 1097916, 714373, 1014728, 4776557, 459609, 1612646, 1378250, 453159, 1529157, 611275, 396167, 740991, 660253, 1563171, 460394, 607832, 712921, 441871, 729424, 820921, 535866, 941588, 440496, 380782, 574597, 365196, 1093868, 959190, 1117959, 365017, 546494, 339343, 595824, 546648, 1660850, 1067032, 1116319, 422346, 446516, 382982, 519606, 1252784, 979687, 1090735, 758062, 843620, 347758, 755997, 1026128, 1319430, 459807, 1308384, 603587, 606305, 632367, 963922, 7262217, 1096258, 1176198, 857066, 1217470, 402148, 618379, 414905, 390148, 564663, 515905, 544384, 1565419, 973071, 1346248, 812372, 423483, 1289298, 1137244, 962874, 771302, 433879, 388136, 1528489, 518205, 1678087, 1266207, 658559, 1069447, 441826, 960142, 501339, 1104544, 1586379, 644501, 922933, 976541, 1156493, 903073, 575234, 406737, 511503, 1085965, 537342, 1715474, 345878, 1285469, 945385, 1516980, 377085, 2357103, 1628300, 991199, 433994, 920884, 712288, 407844, 767159, 382354, 924353, 388640, 643543, 975257, 1408593, 1618251, 541966, 348184, 1231658, 1005491, 493939, 381179, 418001, 428131, 568744, 355722, 565665, 521005, 615288, 1836808, 403824, 599911, 1406292, 1024079, 459829, 1132671, 1326149, 433545, 684012, 596337, 1296487, 757931, 1295343, 1522384, 1662802, 394972, 1070293, 724413, 1045559, 1155259, 1293115, 1032675, 411722, 546291, 403275, 759948, 1306703, 438976, 1114064, 1419736, 1784967, 618412, 339028, 394791, 1771509, 501961, 1482196, 454450, 378575, 599994, 433584, 413539, 429747, 466622, 429928, 446350, 498137, 387920, 931856, 441391, 677353, 612154, 868994, 518851, 493017, 1348693, 1359436, 431486, 1359884, 389945, 1100897, 942350, 1309798, 1036514, 1264291], "NumGC":20701} Allocated: 1730649024, active: 1732280320, metadata: 22460528 (n_thp 0), resident: 1792962560, mapped: 1808351232, retained: 4229349376 2021-09-13T18:50:55.852-07:00 [Info] memstats {"Alloc":92501856, "TotalAlloc":1161622900064, "Sys":283252984, "Lookups":0, "Mallocs":97319638,"Frees":97302307, "HeapAlloc":92501856, "HeapSys":263913472, "HeapIdle":168558592, "HeapInuse":95354880,"HeapReleased":153378816, "HeapObjects":17331,"GCSys":9011200, "LastGC":1631584255795069227,"PauseTotalNs":54236627912, "PauseNs":[443301, 992117, 395911, 515005, 391684, 537590, 413988, 821879, 1207875, 456599, 593123, 598853, 550593, 372489, 1069752, 1382930, 437488, 1483413, 442774, 477875, 1693035, 396344, 853611, 473578, 342975, 450993, 548732, 1131355, 444159, 681863, 610467, 369192, 1213415, 1440222, 512311, 489045, 422462, 888364, 1392890, 4645994, 442767, 565592, 550936, 1061633, 420520, 885586, 503679, 912401, 440764, 534054, 1841776, 916504, 499546, 2188512, 972565, 813008, 351695, 402620, 360930, 980338, 445245, 1580378, 851152, 948759, 896243, 894780, 1120001, 1053830, 1116389, 578161, 450600, 1184182, 470300, 751098, 517185, 431696, 456935, 453943, 1091823, 396496, 325791, 1134491, 667101, 456810, 446846, 390869, 562015, 1336545, 1124029, 602960, 971682, 422411, 540518, 411868, 1288679, 2193347, 809525, 2320855, 825184, 1941876, 552137, 984317, 874867, 1463773, 532196, 509005, 437430, 510245, 418584, 1058420, 448828, 6564361, 936278, 491736, 535179, 427843, 660440, 1026878, 775310, 1029706, 783039, 391982, 527385, 620934, 854934, 633846, 916256, 1732404, 543306, 370391, 1036276, 582580, 621488, 858948, 978338, 438969, 630080, 440903, 1232634, 1172333, 1490639, 923185, 1066777, 462707, 572090, 636763, 926217, 898829, 1455580, 882257, 579640, 392947, 560064, 1317371, 394311, 1877859, 354185, 377192, 387340, 1884188, 1861016, 1326159, 447971, 720415, 720525, 1627498, 1392904, 1150408, 570010, 877284, 1315310, 425700, 1493147, 368640, 455847, 730461, 398679, 883096, 1165590, 351620, 589444, 1047056, 486773, 486230, 1268325, 764172, 804565, 356493, 525859, 487767, 445510, 331091, 1386951, 1198317, 555928, 994165, 563031, 2001109, 669406, 855243, 559930, 1311672, 1247992], "NumGC":20904} Allocated: 1723130712, active: 1724768256, metadata: 22460528 (n_thp 0), resident: 1784287232, mapped: 1798889472, retained: 4238811136 Mon Sep 13 18:51:23 PDT 2021 RSS = 1704968 2021-09-13T18:51:25.913-07:00 [Info] memstats {"Alloc":66753880, "TotalAlloc":1170698791000, "Sys":283252984, "Lookups":0, "Mallocs":97975254,"Frees":97958133, "HeapAlloc":66753880, "HeapSys":263880704, "HeapIdle":194273280, "HeapInuse":69607424,"HeapReleased":167116800, "HeapObjects":17121,"GCSys":9043968, "LastGC":1631584285833347972,"PauseTotalNs":54407396765, "PauseNs":[826340, 482399, 464808, 1220131, 606006, 826492, 505259, 936814, 1865249, 1512142, 1019651, 1032971, 477532, 487605, 734215, 915476, 1002478, 2097672, 583630, 365733, 820808, 351549, 441405, 944999, 535158, 811589, 544203, 1132407, 1137893, 573573, 508075, 570514, 526871, 873893, 402257, 540328, 404057, 750206, 948519, 544421, 511186, 908710, 398302, 343572, 412562, 642254, 2395626, 1494882, 1464794, 414401, 498635, 1306867, 493362, 1282713, 474138, 634500, 734309, 909967, 505779, 922774, 489228, 1463663, 532061, 5797376, 439331, 1522036, 372465, 627886, 587843, 1574609, 709239, 864000, 876471, 1009325, 752911, 928806, 443289, 360359, 675256, 579766, 420394, 446514, 522809, 758660, 1242143, 676220, 389666, 611406, 365557, 589377, 492382, 437487, 1056652, 2661511, 498421, 922445, 469470, 449673, 754301, 937433, 1004071, 486725, 1141794, 558773, 420907, 1054402, 1142124, 778820, 580169, 339236, 449467, 1029992, 514389, 388774, 716118, 622043, 736777, 1265986, 522158, 389551, 398861, 454858, 538808, 1065523, 755029, 938776, 978035, 333929, 762880, 979518, 477615, 992217, 745552, 417040, 1237099, 844021, 1249263, 517012, 1210151, 327148, 1194256, 1277304, 781774, 1652578, 502671, 433930, 725238, 660918, 723937, 1217769, 1345252, 506256, 1212476, 708287, 348816, 1256736, 421634, 426146, 1426702, 1096287, 413670, 866184, 556140, 668445, 536603, 1146308, 394463, 2394298, 428583, 540873, 803161, 464730, 1380910, 487760, 1028827, 579021, 578097, 449698, 499860, 1053830, 630279, 390571, 1291489, 1287125, 923232, 1175714, 519223, 4773597, 685608, 555104, 625915, 522171, 510946, 1099876, 1149688, 718475, 639512, 500990, 1275920, 831522, 567363, 474932, 447300, 434528, 417984, 401360, 430364, 623050, 925874], "NumGC":21113} Allocated: 1730633024, active: 1732255744, metadata: 22460528 (n_thp 0), resident: 1771483136, mapped: 1786085376, retained: 4251615232 2021-09-13 18:51:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 253, "num_active_sctxs": 95, "num_free_sctxs": 158, "num_trim_sctxs": 19, "num_wctxs": 294, "num_free_wctxs": 230, "num_active_wctxs": 64, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 124, "cpu": 12.57658, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1789636608, "memory_jemalloc_allocated": 1730010352, "memory_in_use": 1787342848, "memory_stats_size_total": 1452988412, "memory_stats_size_global": 1077854156, "memory_stats_size_page": 942101044, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 145608, "reclaim_pending": 6687752, "buf_memused": 368421544, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 2, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 14838, "total_ops": 360829, "compacts": 1471207, "purges": 223, "lss_data_size": 260792843, "lss_used_space": 2320664290, "lss_fragmentation": 88, "lss_num_reads": 818099, "lss_read_bs": 128513453165, "lss_blk_read_bs": 125930307584, "rlss_num_reads": 29131, "lss_rdr_reads_bs": 9582288662, "lss_blk_rdr_reads_bs": 3569991680, "bytes_written": 114708897792, "bytes_incoming": 23651619292, "write_amp_avg": 1.86, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 12167, "resident_ratio": 81.99892, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 68795836, "recovery_used_space": 636782982, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86111645696, "recovery_lss_frag": 89, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 593.52000 }2021-09-13 18:51:33 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 18:51:33 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 18:51:35 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 18:51:35 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T18:51:55.956-07:00 [Info] memstats {"Alloc":67410048, "TotalAlloc":1179385563760, "Sys":283252984, "Lookups":0, "Mallocs":98648941,"Frees":98631674, "HeapAlloc":67410048, "HeapSys":264011776, "HeapIdle":193757184, "HeapInuse":70254592,"HeapReleased":160768000, "HeapObjects":17267,"GCSys":8912896, "LastGC":1631584315833555555,"PauseTotalNs":54568028511, "PauseNs":[1026474, 1054860, 541013, 379980, 628992, 1238386, 1224740, 1489323, 1245169, 483056, 1099272, 1337458, 594100, 762872, 750580, 2939598, 1476591, 1408690, 613847, 627265, 1128818, 1551615, 785321, 472819, 1276676, 372682, 778134, 1078180, 426060, 966816, 425864, 435488, 526977, 1090559, 420814, 498443, 876267, 469789, 665659, 401342, 517402, 952806, 650188, 2399348, 354287, 1004792, 694133, 588625, 492870, 1166866, 1821150, 752535, 863528, 1545458, 507968, 700958, 686749, 1223080, 647491, 1751155, 421851, 451294, 921346, 1251733, 913734, 476544, 1193150, 1468905, 356917, 695359, 557115, 893463, 1067712, 992100, 421912, 1277305, 1335126, 482367, 506846, 495453, 555587, 413150, 854891, 719144, 425069, 510938, 331086, 410106, 829417, 814319, 387252, 885449, 827042, 1086130, 497026, 717834, 1238686, 461876, 375826, 428957, 897080, 1488204, 888833, 1198423, 918106, 625610, 393451, 1745958, 578044, 365911, 431477, 1167154, 538146, 1426700, 1022403, 709855, 886850, 841121, 482533, 494608, 536944, 478668, 781352, 550342, 1075395, 796407, 418405, 521171, 1443352, 1195257, 394740, 425391, 621060, 485134, 724829, 490510, 1402027, 719577, 965101, 435537, 465493, 916809, 470485, 490583, 541837, 1038013, 2070168, 551919, 451513, 403975, 1271638, 419458, 499855, 417664, 1476217, 448563, 482065, 843767, 439307, 542737, 808989, 1076291, 1130187, 561035, 457903, 475894, 1043989, 1101834, 599290, 630171, 409475, 631452, 445700, 1147572, 731388, 360008, 491670, 956510, 1638621, 947500, 585304, 786083, 705939, 351324, 459607, 802139, 438156, 1011932, 979604, 391812, 738048, 1742691, 369636, 891518, 497524, 1263710, 1375389, 453016, 1497108], "NumGC":21312} Allocated: 1761674264, active: 1763311616, metadata: 22460528 (n_thp 0), resident: 1811984384, mapped: 1826586624, retained: 4211113984 2021-09-13 18:52:07 [gsi-test] [] { "memory_quota": 994832636, "count": 15578, "compacts": 17699, "purges": 0, "splits": 0, "merges": 0, "inserts": 15578, "deletes": 0, "compact_conflicts": 0, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 0, "persist_conflicts": 4, "memory_size": 106019924, "memory_size_index": 13577836, "allocated": 9670647400, "freed": 9564627476, "reclaimed": 9564627476, "reclaim_pending": 0, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 1, "allocated_index": 13577836, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1408, "num_rec_allocs": 121225, "num_rec_frees": 119817, "num_rec_swapout": 21513, "num_rec_swapin": 21513, "num_rec_compressed": 0, "compresses": 24, "decompresses": 32, "num_compressed_pages": 0, "bytes_incoming": 1021106744, "lss_data_size": 22399716, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 124442866, "est_recovery_size": 57456814, "lss_num_reads": 5671, "lss_read_bs": 1650696307, "lss_blk_read_bs": 1042817024, "lss_rdr_reads_bs": 973672760, "lss_blk_rdr_reads_bs": 379170816, "cache_hits": 64566, "cache_misses": 1359, "cache_hit_ratio": 0.92476, "rlss_num_reads": 3255, "rcache_hits": 49067, "rcache_misses": 1280, "rcache_hit_ratio": 0.90612, "resident_ratio": 1.00000, "mvcc_purge_ratio": 0.00000, "currSn": 363532, "gcSn": 363531, "gcSnIntervals": "[0 211314 240838 362915 363498 363532]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 25, "num_trim_wctxs": 77, "num_readers": 8, "num_writers": 9, "page_bytes": 22417416, "page_cnt": 59, "page_itemcnt": 342, "avg_item_size": 65548, "avg_page_size": 379956, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 119761468, "page_bytes_marshalled":13398665220, "page_bytes_compressed":7296468990, "compression_ratio": 1.83632, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 17736, "num_total_smr": 1, "num_active_smr": 0, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 18986202, "lss_data_size": 32570720, "lss_used_space": 182236783, "lss_disk_size": 236728320, "lss_fragmentation": 82, "lss_num_reads": 149758, "lss_read_bs": 21589985185, "lss_blk_read_bs": 21805355008, "bytes_written": 19778019328, "bytes_incoming": 1021106744, "write_amp": 5.68, "write_amp_avg": 2.74, "lss_gc_num_reads": 146424, "lss_gc_reads_bs": 20509402228, "lss_blk_gc_reads_bs": 21337489408, "lss_gc_status": "frag 30, data: 44830072, used: 64579603, disk: 69410983, relocated: 1560632, retries: 935, skipped: 140864, cleaned:1164485465, log:(10430769014 - 10495598592), disk:(10426187609-10495598592), activeFragRatio:30, activeMaxFragRatio:80, run:189 duration:124697 ms elapsed: 292043 ms", "lss_head_offset": 10430769014, "lss_tail_offset": 10555449344, "num_sctxs": 35, "num_free_sctxs": 27, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } 2021-09-13 18:52:11 [gsi-test] [] { "memory_quota": 994832636, "count": 14922, "compacts": 16508, "purges": 0, "splits": 0, "merges": 0, "inserts": 14922, "deletes": 0, "compact_conflicts": 1, "split_conflicts": 0, "merge_conflicts": 0, "insert_conflicts": 0, "delete_conflicts": 0, "swapin_conflicts": 77, "persist_conflicts": 17, "memory_size": 103855856, "memory_size_index": 13577820, "allocated": 9737373832, "freed": 9633517976, "reclaimed": 9632993480, "reclaim_pending": 524496, "reclaim_list_size": 0, "reclaim_list_count": 0, "reclaim_threshold": 0, "allocated_index": 13577820, "freed_index": 0, "reclaimed_index": 0, "num_pages": 208, "items_count": 0, "total_records": 1382, "num_rec_allocs": 119552, "num_rec_frees": 118175, "num_rec_swapout": 31906, "num_rec_swapin": 31901, "num_rec_compressed": 0, "compresses": 0, "decompresses": 0, "num_compressed_pages": 0, "bytes_incoming": 978107256, "lss_data_size": 20005252, "lss_recoverypt_size": 4096, "lss_maxsn_size": 4096, "checkpoint_used_space":0, "est_disk_size": 142894657, "est_recovery_size": 66112186, "lss_num_reads": 8491, "lss_read_bs": 2440900725, "lss_blk_read_bs": 1572679680, "lss_rdr_reads_bs": 1432187173, "lss_blk_rdr_reads_bs": 596979712, "cache_hits": 60864, "cache_misses": 2228, "cache_hit_ratio": 0.87031, "rlss_num_reads": 4855, "rcache_hits": 46108, "rcache_misses": 2062, "rcache_hit_ratio": 0.82160, "resident_ratio": 0.99638, "mvcc_purge_ratio": 0.00000, "currSn": 363533, "gcSn": 363531, "gcSnIntervals": "[0 212424 241944 362914 363497 363532]", "purger_running": false, "mem_throttled": false, "lss_throttled": false, "num_wctxs": 9, "num_free_wctxs": 29, "num_trim_wctxs": 52, "num_readers": 11, "num_writers": 11, "page_bytes": 25957008, "page_cnt": 65, "page_itemcnt": 396, "avg_item_size": 65548, "avg_page_size": 399338, "act_max_page_items": 4, "act_min_page_items": 1, "act_max_delta_len": 2, "est_resident_mem": 137385139, "page_bytes_marshalled":13695870896, "page_bytes_compressed":7458860334, "compression_ratio": 1.83619, "bloom_tests": 0, "bloom_negatives": 0, "bloom_false_positives":0, "bloom_fp_rate": 0.00000, "memory_size_bloom_filter":0, "memory_size_delta": 16960, "num_total_smr": 1, "num_active_smr": 1, "num_pending_smr": 0, "lss_stats": { "shared": false, "punch_hole_support": true, "buf_memused": 30587554, "lss_data_size": 29323916, "lss_used_space": 211127978, "lss_disk_size": 297103360, "lss_fragmentation": 86, "lss_num_reads": 144918, "lss_read_bs": 21539779442, "lss_blk_read_bs": 21441929216, "bytes_written": 18979160064, "bytes_incoming": 978107256, "write_amp": 7.62, "write_amp_avg": 3.16, "lss_gc_num_reads": 139897, "lss_gc_reads_bs": 19983960329, "lss_blk_gc_reads_bs": 20759695360, "lss_gc_status": "frag 32, data: 56970251, used: 84717576, disk: 87152487, relocated: 1222481, retries: 1281, skipped: 143323, cleaned:1392160767, log:(11429044216 - 11513761792), disk:(11426609305-11513761792), activeFragRatio:30, activeMaxFragRatio:80, run:147 duration:145527 ms elapsed: 292250 ms", "lss_head_offset": 11429044216, "lss_tail_offset": 11573481472, "num_sctxs": 40, "num_free_sctxs": 31, "num_trim_sctxs": 0, "num_swapperWriter": 32 } } Mon Sep 13 18:52:23 PDT 2021 RSS = 1741488 2021-09-13T18:52:25.991-07:00 [Info] memstats {"Alloc":84387824, "TotalAlloc":1187668002544, "Sys":283252984, "Lookups":0, "Mallocs":99321600,"Frees":99304348, "HeapAlloc":84387824, "HeapSys":263946240, "HeapIdle":176676864, "HeapInuse":87269376,"HeapReleased":163143680, "HeapObjects":17252,"GCSys":8978432, "LastGC":1631584345946149803,"PauseTotalNs":54711664749, "PauseNs":[546890, 806752, 461282, 454289, 1175720, 390260, 808460, 442193, 395221, 1242746, 484267, 1075589, 526580, 846134, 818065, 1034143, 458090, 1765977, 1006843, 397624, 1117179, 932513, 485623, 1408748, 471163, 1425275, 941704, 395585, 401223, 766243, 1349824, 466972, 342478, 460009, 608628, 722724, 795173, 924119, 408576, 448014, 1016800, 973655, 969110, 371647, 986270, 632117, 1211059, 474389, 429546, 459209, 991265, 462109, 431275, 349319, 803076, 440514, 1262083, 1169166, 529699, 352870, 444720, 893399, 412461, 747601, 402811, 1390745, 1211159, 1000345, 410230, 1114274, 867944, 570294, 626259, 1045337, 382692, 1444592, 670406, 415858, 945943, 427766, 384834, 392548, 354171, 1152636, 1557200, 357201, 859404, 667431, 385627, 482415, 437479, 1214005, 417031, 329239, 882448, 543986, 1320315, 1889112, 862421, 511126, 837712, 477375, 555181, 723171, 520766, 1369907, 961687, 884779, 471487, 829990, 494554, 1246751, 1078694, 345123, 1344110, 940372, 1217896, 755401, 607651, 531080, 1449058, 963853, 690719, 789498, 463516, 495615, 439287, 432151, 1105312, 1021310, 629782, 631542, 643301, 386995, 456157, 690638, 567382, 1118845, 510990, 457612, 787151, 412253, 374372, 392487, 412653, 857032, 413357, 357624, 1391207, 440773, 754041, 800852, 816995, 390610, 359844, 471465, 589029, 817827, 412360, 472355, 759550, 1290473, 866906, 1250605, 402671, 965540, 1028787, 440210, 860525, 800292, 857389, 1440153, 1208277, 848517, 591015, 942089, 467815, 951939, 414455, 734517, 522119, 478154, 1284117, 1322344, 471214, 401838, 452125, 1619930, 412639, 742798, 397842, 693011, 643392, 461912], "NumGC":21506} Allocated: 1779114808, active: 1780744192, metadata: 22460528 (n_thp 0), resident: 1856663552, mapped: 1871265792, retained: 4166434816 2021-09-13T18:52:56.032-07:00 [Info] memstats {"Alloc":80573768, "TotalAlloc":1195768009296, "Sys":283252984, "Lookups":0, "Mallocs":99984259,"Frees":99966982, "HeapAlloc":80573768, "HeapSys":263913472, "HeapIdle":180486144, "HeapInuse":83427328,"HeapReleased":148660224, "HeapObjects":17277,"GCSys":9011200, "LastGC":1631584375957380843,"PauseTotalNs":54863788696, "PauseNs":[1163280, 920817, 369263, 512915, 1246375, 507492, 438698, 978144, 399565, 493259, 807708, 593102, 1013420, 460789, 1282648, 443997, 394955, 536004, 914287, 2203722, 587553, 1544010, 1379773, 861051, 1846410, 1383784, 403657, 820347, 1385244, 540036, 1053459, 1222803, 412988, 806895, 433901, 787507, 394291, 797616, 1299071, 887494, 994309, 778422, 967352, 498889, 570542, 1068516, 380049, 938124, 1898933, 635757, 438431, 1345284, 753208, 1433673, 624428, 1179439, 1018295, 1707648, 1127809, 759752, 416935, 468417, 756974, 971024, 1016905, 519415, 381445, 672055, 415999, 1051021, 1196677, 770896, 1291514, 398514, 860545, 870932, 652368, 558378, 820931, 2164562, 1067582, 458957, 1029381, 1018478, 1469451, 1712887, 489726, 483315, 592480, 471180, 1266425, 470135, 903062, 424600, 365350, 614386, 426853, 432653, 913070, 928760, 600247, 877083, 415239, 491031, 406114, 928137, 1019464, 570295, 1052029, 530914, 836126, 845642, 1186593, 467252, 931659, 898206, 1012881, 665299, 468807, 476755, 490929, 943775, 465070, 392225, 1085134, 1026785, 409071, 767202, 374928, 367009, 731298, 1773257, 543034, 523868, 309381, 1208533, 424578, 482720, 569338, 772001, 383291, 768200, 1251726, 448287, 1085809, 589292, 418727, 418354, 438026, 466672, 1836256, 540834, 652885, 561846, 550321, 475397, 442130, 713733, 1205396, 1723733, 1401957, 387142, 707189, 1508570, 1457842, 778964, 407635, 431416, 1030818, 1060940, 1689555, 916098, 1470694, 409814, 961519, 414072, 876059, 815999, 484610, 469476, 1292896, 450229, 678301, 578147, 489187, 416788, 473392, 314035, 646295, 352056], "NumGC":21696} Allocated: 1801761968, active: 1803386880, metadata: 22460528 (n_thp 0), resident: 1870643200, mapped: 1885245440, retained: 4152455168 Mon Sep 13 18:53:23 PDT 2021 RSS = 1771688 2021-09-13T18:53:26.107-07:00 [Info] memstats {"Alloc":73464048, "TotalAlloc":1204398599256, "Sys":283252984, "Lookups":0, "Mallocs":100692940,"Frees":100675105, "HeapAlloc":73464048, "HeapSys":264011776, "HeapIdle":187686912, "HeapInuse":76324864,"HeapReleased":133603328, "HeapObjects":17835,"GCSys":8912896, "LastGC":1631584405991780380,"PauseTotalNs":55021991602, "PauseNs":[593530, 811870, 984014, 437624, 911769, 485477, 462539, 397559, 436204, 529930, 406911, 744730, 1187641, 420544, 655056, 452177, 445187, 787865, 2014615, 1464128, 692354, 441424, 624828, 1584341, 500762, 1339702, 997429, 1267229, 413269, 1291230, 429882, 367962, 1199401, 962372, 486794, 1075476, 357444, 406486, 585494, 554488, 702834, 683804, 350205, 362202, 431131, 361091, 377386, 1841882, 1412402, 971739, 1622396, 4927249, 1105863, 1055616, 1360817, 439577, 351016, 637641, 526546, 1412786, 439327, 1071429, 903806, 414648, 737248, 1131101, 410826, 389272, 1626243, 368399, 808026, 1161409, 1173267, 1275223, 887858, 516018, 533206, 438817, 1316381, 725037, 696311, 1335063, 1234915, 420482, 411019, 383195, 495911, 1008204, 1084408, 1366306, 471532, 423577, 403022, 589605, 450184, 737486, 485148, 518197, 656281, 752692, 443094, 1638083, 455107, 519809, 594607, 715225, 917103, 1011375, 1779491, 1004139, 386109, 371764, 498292, 529021, 333684, 858314, 515354, 1923640, 574629, 778282, 910790, 400265, 882180, 585414, 492059, 1098922, 1642905, 744489, 1123270, 853801, 507461, 390416, 429411, 773573, 774704, 708744, 425212, 843365, 1119349, 1155634, 1581515, 1668267, 383095, 420508, 497869, 497234, 1054603, 1002631, 751209, 513727, 414257, 619283, 1061663, 977860, 1342405, 357863, 1033643, 446563, 683733, 1156544, 347946, 992035, 406565, 436050, 428920, 697687, 558590, 556962, 471492, 611695, 420974, 952035, 969612, 470009, 1242527, 417764, 886326, 473875, 333811, 1254402, 1419377, 671419, 1144167, 418027, 1015847, 671865, 546684, 1686977, 523966, 355028, 587087, 983599, 516288, 1565898, 462637, 1198486, 746468, 1193252], "NumGC":21894} Allocated: 1815358752, active: 1816969216, metadata: 22460528 (n_thp 0), resident: 1907617792, mapped: 1922220032, retained: 4115480576 2021-09-13T18:53:56.144-07:00 [Info] memstats {"Alloc":70507888, "TotalAlloc":1212693300104, "Sys":283252984, "Lookups":0, "Mallocs":101372947,"Frees":101356466, "HeapAlloc":70507888, "HeapSys":263913472, "HeapIdle":190496768, "HeapInuse":73416704,"HeapReleased":157081600, "HeapObjects":16481,"GCSys":9011200, "LastGC":1631584436116957110,"PauseTotalNs":55184238614, "PauseNs":[429869, 519858, 422110, 848414, 542766, 451504, 517617, 1446574, 1201729, 1785592, 813228, 374107, 984546, 602646, 449761, 421511, 396820, 399111, 846117, 353053, 349079, 1424161, 556282, 377165, 390888, 1994321, 1154355, 450420, 1766687, 1784960, 595613, 1209848, 366364, 476347, 485702, 497273, 878803, 1279318, 602136, 650594, 479231, 1571553, 2900960, 341254, 467089, 588757, 1199669, 358299, 863968, 1263061, 996029, 518116, 517134, 397593, 773729, 1668143, 497239, 4246648, 619546, 2041436, 1914827, 484006, 502098, 950582, 459126, 601363, 535995, 404363, 469374, 1199833, 354933, 427790, 668421, 709418, 752219, 456656, 478865, 998821, 662649, 485226, 651954, 7163160, 628846, 1479422, 853041, 635050, 429094, 305340, 770630, 868508, 500082, 633299, 458180, 535339, 396485, 588340, 931909, 493186, 394234, 508232, 447212, 461318, 452552, 762373, 1424284, 353127, 968061, 6401166, 609762, 430217, 988265, 486436, 1174488, 1003921, 639998, 937349, 626431, 946377, 532591, 1194807, 980446, 513741, 1483260, 1686721, 549162, 790312, 1046635, 442810, 954270, 268728, 1232512, 486729, 1139908, 390261, 514410, 627278, 736467, 639063, 532103, 405492, 405535, 390925, 520249, 910995, 1058725, 824801, 1004257, 384965, 510697, 447505, 916612, 516580, 393352, 1094454, 605246, 921156, 396292, 818967, 1984412, 432825, 504387, 428627, 532520, 385144, 1135998, 1405440, 829441, 628977, 386459, 948933, 626776, 445034, 484227, 582284, 1185626, 489266, 476992, 516317, 480280, 532618, 407631, 1266802, 592662, 359587, 1037832, 1459371, 1601995, 840508, 608594, 729630, 485517, 454917, 1259282, 622135, 568011], "NumGC":22089} Allocated: 1821725008, active: 1823338496, metadata: 22460528 (n_thp 0), resident: 1886744576, mapped: 1901346816, retained: 4136353792 Mon Sep 13 18:54:23 PDT 2021 RSS = 1779048 2021-09-13T18:54:26.184-07:00 [Info] memstats {"Alloc":67754904, "TotalAlloc":1220991719888, "Sys":283252984, "Lookups":0, "Mallocs":102053462,"Frees":102036189, "HeapAlloc":67754904, "HeapSys":263946240, "HeapIdle":193314816, "HeapInuse":70631424,"HeapReleased":170319872, "HeapObjects":17273,"GCSys":8978432, "LastGC":1631584466130082581,"PauseTotalNs":55336143334, "PauseNs":[929182, 713116, 1815697, 1029254, 559361, 777423, 434439, 756887, 1070492, 605693, 1221030, 552038, 498064, 403497, 376443, 445120, 437032, 1270973, 495275, 1370627, 515036, 457144, 394843, 504396, 616663, 333602, 386263, 576204, 463886, 422700, 1673300, 843396, 797025, 854128, 1409309, 934103, 2064151, 725087, 327939, 430083, 468801, 798658, 1306883, 454163, 609298, 375522, 592177, 1145252, 1068486, 385190, 1403794, 1296625, 776499, 343199, 657653, 1174700, 3602152, 455216, 415565, 1087881, 514380, 1069077, 1103849, 760300, 893476, 818945, 410510, 355302, 444733, 339360, 373064, 1243085, 892439, 1005611, 1203796, 487886, 1332428, 415864, 517074, 915469, 445264, 325189, 434254, 1293430, 729614, 544302, 1156975, 541662, 1046396, 1116380, 1594874, 575933, 527518, 513485, 410809, 582633, 451152, 988885, 1015141, 416959, 751575, 448870, 1189810, 846172, 536500, 487200, 675288, 378049, 1162797, 1089156, 457990, 2060140, 1021901, 1219597, 464120, 403418, 1038085, 2358270, 401604, 883504, 579379, 630650, 1165992, 621418, 622129, 332400, 513068, 784945, 524046, 344163, 489502, 663235, 618077, 371526, 815948, 447826, 687676, 504224, 810694, 1299601, 449697, 1321779, 916595, 751985, 890044, 600103, 536482, 899027, 599939, 410885, 875577, 1230785, 1389900, 480292, 700014, 1674389, 455906, 1445405, 477655, 419371, 615583, 1111519, 554261, 1106902, 436720, 532592, 518565, 464881, 1423378, 444019, 1931464, 435521, 344476, 548593, 1024546, 1369022, 436959, 1072517, 441322, 439343, 511729, 791597, 756336, 1507777, 334119, 800469, 490884, 810108, 973871, 465168, 2110268, 410235, 747483], "NumGC":22282} Allocated: 1823854424, active: 1825480704, metadata: 22460528 (n_thp 0), resident: 1902563328, mapped: 1917165568, retained: 4120535040 2021-09-13T18:54:56.221-07:00 [Info] memstats {"Alloc":70644352, "TotalAlloc":1229493788256, "Sys":283252984, "Lookups":0, "Mallocs":102753187,"Frees":102735964, "HeapAlloc":70644352, "HeapSys":263946240, "HeapIdle":190423040, "HeapInuse":73523200,"HeapReleased":156336128, "HeapObjects":17223,"GCSys":8978432, "LastGC":1631584496178139063,"PauseTotalNs":55521648169, "PauseNs":[421234, 1655206, 1337658, 596624, 1028835, 451044, 440541, 1432878, 1281605, 1091685, 1406866, 393236, 367451, 528051, 540651, 1218993, 578133, 1505334, 564284, 326858, 410958, 497008, 491624, 378637, 1322864, 517673, 972365, 1653294, 720548, 390305, 480580, 686852, 489314, 393710, 501705, 448966, 374148, 539992, 735997, 536192, 580144, 670797, 589916, 493860, 840840, 1244459, 753720, 380981, 541708, 549640, 1110140, 1123510, 403303, 544292, 1032066, 475376, 639904, 576717, 536138, 1585599, 1550966, 673493, 594289, 1486700, 670512, 870096, 985669, 834471, 954859, 663503, 1049626, 1146956, 401382, 420737, 1378007, 360289, 380561, 1119248, 514813, 948058, 433496, 513322, 1496919, 359171, 369175, 1442057, 661772, 405899, 546387, 466418, 492578, 886198, 327959, 387001, 507431, 4838393, 1225187, 1249732, 1086990, 965593, 938628, 1316646, 1417155, 1040981, 1188803, 351508, 8706482, 1284907, 423487, 351410, 382475, 374802, 556807, 441369, 1020408, 577161, 629154, 407873, 515672, 986724, 608331, 446406, 1008971, 20022080, 621838, 1251466, 402138, 426084, 442867, 476546, 320305, 1110378, 845299, 803303, 906190, 891038, 1565696, 324825, 662362, 525148, 920498, 1089185, 1071456, 956280, 1194132, 578116, 507338, 394690, 1238899, 416162, 726936, 1395912, 2126845, 901396, 410171, 880824, 496912, 419654, 738886, 637524, 761979, 479805, 398193, 1587367, 477329, 1037534, 2088270, 389274, 469082, 1224016, 1044658, 637487, 1330382, 1279378, 1048813, 1247326, 440609, 671733, 837803, 464344, 450113, 1709406, 1345536, 802515, 978172, 1463886, 550367, 1402775, 995471, 1153951, 1258753, 939124, 420437, 394486], "NumGC":22476} Allocated: 1824679528, active: 1826308096, metadata: 22460528 (n_thp 0), resident: 1895624704, mapped: 1910226944, retained: 4127473664 Mon Sep 13 18:55:23 PDT 2021 RSS = 1777308 2021-09-13T18:55:26.253-07:00 [Info] memstats {"Alloc":73545624, "TotalAlloc":1238010703592, "Sys":283252984, "Lookups":0, "Mallocs":103448309,"Frees":103430272, "HeapAlloc":73545624, "HeapSys":264011776, "HeapIdle":187629568, "HeapInuse":76382208,"HeapReleased":159031296, "HeapObjects":18037,"GCSys":8912896, "LastGC":1631584526058294511,"PauseTotalNs":55669124414, "PauseNs":[977060, 1236621, 392736, 1675846, 720515, 1144000, 960761, 519107, 386862, 827343, 392242, 469717, 518989, 1085099, 436438, 485592, 1148658, 1363683, 1727251, 585911, 888983, 366090, 990433, 686764, 1297224, 566917, 508990, 2817684, 391442, 420378, 356952, 324026, 1253134, 761837, 385905, 529232, 812944, 1186244, 481234, 443220, 490343, 542628, 366292, 1052904, 611889, 427672, 1066856, 444987, 734988, 1095379, 508518, 489273, 428340, 532376, 477492, 831771, 414158, 997727, 452241, 461156, 923101, 1175895, 567987, 1499680, 1135140, 559657, 427167, 1196330, 745941, 721287, 448937, 404782, 479036, 1098074, 627723, 419300, 753507, 434446, 518236, 532607, 539514, 400151, 941030, 997735, 416468, 1345696, 446635, 1313967, 390818, 412557, 528435, 404783, 1109573, 412650, 703755, 557618, 406448, 421840, 1086926, 531475, 398303, 929523, 711153, 370698, 985052, 983060, 517770, 1014129, 454084, 1557286, 446468, 448913, 1014916, 1901455, 403892, 628715, 598304, 439196, 459774, 1501326, 1076516, 492809, 450717, 1019678, 523272, 395501, 1095872, 1302681, 808743, 587557, 990508, 1016567, 445245, 1364836, 512235, 1620865, 400031, 729416, 518426, 492291, 1814405, 625006, 1151614, 530276, 596017, 1024300, 470094, 602003, 656129, 394131, 1348500, 661585, 866976, 1190637, 1283754, 463621, 609981, 417248, 578455, 624070, 702157, 480690, 1162940, 428164, 461885, 527679, 422520, 426658, 498712, 822465, 1095756, 478800, 398182, 517948, 432091, 394664, 1066352, 596887, 392881, 655096, 1914861, 409060, 535452, 582541, 804691, 1146344, 497098, 1447526, 521262, 401368, 1199333, 1714105, 387718, 1106107, 912493, 1054765, 374448], "NumGC":22673} Allocated: 1824943016, active: 1826549760, metadata: 22460528 (n_thp 0), resident: 1894916096, mapped: 1909518336, retained: 4128182272 2021-09-13T18:55:56.297-07:00 [Info] memstats {"Alloc":77710784, "TotalAlloc":1246724170912, "Sys":283252984, "Lookups":0, "Mallocs":104165768,"Frees":104147862, "HeapAlloc":77710784, "HeapSys":263880704, "HeapIdle":183345152, "HeapInuse":80535552,"HeapReleased":168493056, "HeapObjects":17906,"GCSys":9043968, "LastGC":1631584556243785297,"PauseTotalNs":55835337743, "PauseNs":[1647667, 1972972, 893344, 1108422, 812005, 1004267, 405387, 470706, 993750, 890196, 593400, 808682, 1364030, 388110, 429525, 960897, 1193137, 552341, 2367914, 405429, 436879, 1567037, 1563509, 707750, 762806, 411823, 379678, 623263, 876849, 540943, 616880, 825898, 1091684, 567196, 713107, 1235117, 1302313, 366844, 433128, 930983, 624863, 828126, 494418, 1263494, 763939, 1023904, 424558, 898219, 1237614, 1399196, 1064158, 739705, 1109300, 418040, 1755533, 478306, 495747, 528208, 797222, 384898, 1063842, 1261405, 1074099, 427714, 602874, 410220, 2204273, 640734, 467474, 1030127, 544614, 573146, 385311, 935642, 794495, 1135511, 549024, 581796, 703524, 1471099, 903614, 492455, 668726, 787684, 443981, 1112602, 474424, 405369, 489559, 2310444, 1052234, 536842, 716554, 560746, 1232720, 620123, 363470, 896059, 385649, 975198, 1214111, 373529, 747921, 884095, 434909, 855011, 1383435, 772744, 660089, 1329465, 1413894, 406211, 1315684, 1379509, 812931, 716627, 356088, 844724, 1033904, 382475, 484000, 1184773, 871165, 416863, 619464, 394648, 431855, 449858, 2033304, 665475, 884321, 1809343, 531350, 363521, 406536, 740475, 578211, 539538, 369259, 1341935, 577148, 458956, 434858, 823191, 371452, 3540715, 675295, 600896, 921706, 382951, 442300, 1387819, 1097437, 529284, 791810, 961461, 940709, 642767, 548018, 597565, 497715, 982487, 650273, 692481, 1423716, 634976, 1080435, 969382, 1255859, 445785, 417567, 809734, 417448, 1135731, 922229, 792772, 646965, 464825, 945265, 829963, 596953, 2332923, 1563912, 694356, 409254, 408229, 927747, 550739, 645767, 860381, 768955, 443974, 1067209, 490979, 445002, 1364143, 421167, 326116, 1487417, 1002217], "NumGC":22873} Allocated: 1827211832, active: 1828827136, metadata: 22460528 (n_thp 0), resident: 1890639872, mapped: 1905242112, retained: 4132458496 Mon Sep 13 18:56:23 PDT 2021 RSS = 1799176 2021-09-13T18:56:26.345-07:00 [Info] memstats {"Alloc":68142976, "TotalAlloc":1255183065320, "Sys":283252984, "Lookups":0, "Mallocs":104861155,"Frees":104844087, "HeapAlloc":68142976, "HeapSys":263946240, "HeapIdle":192962560, "HeapInuse":70983680,"HeapReleased":157638656, "HeapObjects":17068,"GCSys":8978432, "LastGC":1631584586295276063,"PauseTotalNs":55986975868, "PauseNs":[524752, 941992, 462370, 645076, 562811, 595723, 464208, 574564, 1090139, 1092031, 394203, 574010, 821272, 1379748, 402958, 435844, 403715, 961285, 891759, 864913, 1254489, 586973, 651116, 715250, 1181264, 469340, 1375457, 935548, 1776869, 709106, 648309, 908669, 486569, 1070962, 458500, 463884, 917816, 1069957, 405887, 422424, 574569, 425239, 663907, 408471, 371121, 340139, 464286, 437851, 419788, 518499, 401344, 607717, 692689, 985628, 864497, 536511, 640367, 1041369, 1238062, 1853193, 921079, 1347099, 1345818, 482342, 409027, 1345065, 1133741, 906295, 575527, 1199262, 408146, 610197, 433096, 398317, 933177, 849750, 875526, 834672, 443821, 500535, 1270822, 420773, 406151, 1183118, 1168452, 619908, 564324, 1949558, 1087669, 837152, 820596, 387199, 870807, 1914262, 337054, 820267, 351953, 1789060, 615431, 359330, 415708, 447907, 407757, 529952, 610440, 1877730, 592348, 1004584, 381689, 576032, 659640, 431468, 988385, 374227, 493636, 529377, 1073806, 997674, 1418212, 427547, 928282, 414336, 1714321, 407283, 423578, 1154653, 554919, 1077745, 801581, 978868, 416588, 1219172, 724456, 452106, 621301, 522506, 493273, 489829, 917349, 481381, 1877258, 1359669, 740770, 1460196, 472783, 1215181, 610948, 717118, 369970, 396823, 589328, 446556, 433674, 1151013, 939418, 1006687, 800023, 842934, 938669, 440536, 355586, 426626, 350939, 787988, 1084975, 1282292, 392561, 1450018, 1236612, 2142938, 473334, 556737, 894227, 505352, 686075, 651879, 413795, 442525, 657476, 727137, 908362, 440467, 1004421, 403739, 1223288, 857381, 1305101, 353275, 662318, 398608, 955150, 767073, 956466, 515638, 1168171, 555008], "NumGC":23069} Allocated: 1826809808, active: 1828429824, metadata: 22460528 (n_thp 0), resident: 1897451520, mapped: 1912053760, retained: 4125646848 2021-09-13 18:56:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 265, "num_active_sctxs": 93, "num_free_sctxs": 172, "num_trim_sctxs": 19, "num_wctxs": 303, "num_free_wctxs": 240, "num_active_wctxs": 63, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 125, "cpu": 8.34400, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1904660480, "memory_jemalloc_allocated": 1838933928, "memory_in_use": 1904660480, "memory_stats_size_total": 1532702288, "memory_stats_size_global": 1187078432, "memory_stats_size_page": 1051378440, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 140040, "reclaim_pending": 6949640, "buf_memused": 338596136, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 20, "num_active_smr": 1, "num_pending_smr": 9, "num_pages": 2080, "items_count": 25, "total_records": 16500, "total_ops": 387465, "compacts": 1605350, "purges": 223, "lss_data_size": 260411622, "lss_used_space": 2668271330, "lss_fragmentation": 90, "lss_num_reads": 818101, "lss_read_bs": 128513953355, "lss_blk_read_bs": 125930561536, "rlss_num_reads": 29132, "lss_rdr_reads_bs": 9582538757, "lss_blk_rdr_reads_bs": 3570245632, "bytes_written": 115056504832, "bytes_incoming": 25397555820, "write_amp_avg": 1.64, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 13834, "resident_ratio": 83.84242, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 68664704, "recovery_used_space": 748976518, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86223839232, "recovery_lss_frag": 90, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 660.00000 }2021-09-13T18:56:56.374-07:00 [Info] memstats {"Alloc":68540288, "TotalAlloc":1263711652408, "Sys":283252984, "Lookups":0, "Mallocs":105560132,"Frees":105542495, "HeapAlloc":68540288, "HeapSys":264011776, "HeapIdle":192659456, "HeapInuse":71352320,"HeapReleased":160432128, "HeapObjects":17637,"GCSys":8912896, "LastGC":1631584616267184827,"PauseTotalNs":56138313380, "PauseNs":[1066807, 423673, 383475, 1527797, 402524, 514229, 543575, 986814, 934343, 845372, 850867, 432594, 746753, 489118, 453737, 534068, 462604, 378735, 469169, 1434729, 369694, 590447, 1338604, 501393, 452187, 583308, 2272256, 1035798, 512468, 459804, 446885, 986025, 691363, 443057, 782363, 539590, 345876, 352323, 1484746, 396915, 1083939, 933713, 553414, 1110086, 480950, 584128, 1190875, 843126, 512808, 768550, 1614931, 407227, 968065, 478234, 1218313, 673045, 453893, 910490, 1240836, 1398864, 744500, 989215, 418518, 893983, 469046, 838689, 507883, 495982, 792276, 498872, 382407, 1204414, 456230, 1369381, 750966, 1194062, 507289, 499524, 824798, 895315, 510739, 1002255, 1062797, 410738, 1430758, 537297, 789960, 476753, 1740891, 1364865, 1321414, 445513, 502449, 1408862, 396734, 1425903, 472186, 463661, 1631303, 667077, 1069109, 567116, 1007321, 418934, 403367, 495851, 468653, 401258, 582825, 931579, 1099528, 778646, 397535, 536219, 599512, 348825, 585486, 969607, 443914, 611289, 568542, 1033437, 1245839, 555385, 1011257, 1147859, 450190, 366276, 482490, 456424, 469332, 655638, 425918, 1302297, 881283, 424123, 983465, 1000030, 786485, 815211, 774586, 490114, 802921, 1469072, 305646, 645135, 752824, 413424, 426610, 637650, 472067, 349835, 571374, 640063, 681903, 2472786, 1023976, 2487652, 1063887, 526456, 363142, 771335, 1463939, 1326698, 659774, 968898, 1258719, 890960, 1234228, 437203, 557135, 1191953, 1294454, 468691, 1337402, 371346, 1286220, 1020272, 392891, 373505, 600440, 419076, 601686, 342562, 467852, 917993, 409494, 983945, 864940, 618875, 445828, 546710, 446221, 622833, 1578399, 328780], "NumGC":23265} Allocated: 1828266320, active: 1829892096, metadata: 22460528 (n_thp 0), resident: 1872797696, mapped: 1887399936, retained: 4150300672 Mon Sep 13 18:57:24 PDT 2021 RSS = 1806256 2021-09-13T18:57:26.401-07:00 [Info] memstats {"Alloc":76835248, "TotalAlloc":1272593465640, "Sys":283252984, "Lookups":0, "Mallocs":106290791,"Frees":106273060, "HeapAlloc":76835248, "HeapSys":263946240, "HeapIdle":184295424, "HeapInuse":79650816,"HeapReleased":169746432, "HeapObjects":17731,"GCSys":8978432, "LastGC":1631584646298233787,"PauseTotalNs":56294161518, "PauseNs":[708253, 639660, 1288969, 495889, 391065, 407825, 850729, 394426, 1232848, 1046073, 1980959, 2066801, 540618, 493115, 501844, 521455, 1458199, 1037233, 583877, 1346517, 425848, 525327, 802032, 1433196, 1447780, 391564, 415710, 362703, 669459, 416023, 441505, 532459, 497702, 349425, 889843, 433050, 1078123, 336047, 825718, 973031, 378646, 1009117, 416712, 1005485, 533196, 1907706, 1066580, 1026231, 400761, 392579, 460724, 1525989, 509463, 451271, 389691, 1484662, 502940, 1325988, 469419, 623302, 387873, 500188, 681093, 987856, 1889705, 513924, 509837, 721800, 357418, 416755, 408399, 1227163, 1083859, 1451172, 583678, 540577, 876328, 1149149, 627955, 778980, 738360, 571665, 306498, 378293, 574612, 428567, 1288257, 812288, 422742, 376475, 1241147, 363930, 389488, 1459906, 834368, 392643, 313081, 561867, 383106, 672456, 327421, 435342, 925213, 917484, 400017, 1195680, 733671, 510766, 1258057, 1410818, 893554, 841777, 374702, 1003733, 481982, 500445, 3143087, 633249, 410246, 434658, 941519, 1117356, 441062, 1303451, 430126, 884733, 443664, 415722, 343401, 751484, 1504939, 530241, 1059025, 1161687, 1114234, 548573, 956174, 579506, 434181, 446863, 567757, 455887, 522067, 424635, 1833234, 553251, 415614, 1315736, 555318, 1662799, 1356331, 497632, 613916, 1255681, 405235, 485702, 397379, 1029544, 980951, 395312, 594724, 435950, 753637, 792486, 423615, 927844, 959021, 602129, 683501, 453070, 347498, 442771, 884378, 404318, 426320, 365437, 426749, 532611, 323924, 350039, 336353, 765398, 510780, 1220350, 1160021, 2170821, 823861, 406005, 467183, 2745582, 508683, 403388, 1164918, 409847, 402944, 1917157, 340005, 1713872, 625260, 913849, 513521, 455959, 1079727, 381988], "NumGC":23469} Allocated: 1831759464, active: 1833365504, metadata: 22460528 (n_thp 0), resident: 1885413376, mapped: 1900015616, retained: 4137684992 2021-09-13T18:57:56.434-07:00 [Info] memstats {"Alloc":71531872, "TotalAlloc":1281397783096, "Sys":283252984, "Lookups":0, "Mallocs":107014515,"Frees":106997658, "HeapAlloc":71531872, "HeapSys":263880704, "HeapIdle":189464576, "HeapInuse":74416128,"HeapReleased":163520512, "HeapObjects":16857,"GCSys":9043968, "LastGC":1631584676369852223,"PauseTotalNs":56447656901, "PauseNs":[506016, 466625, 1013893, 891728, 474337, 570487, 408365, 769897, 453680, 844892, 446696, 385665, 974289, 407424, 435083, 458216, 440217, 671444, 1246034, 605723, 1052856, 505314, 759444, 937902, 1213445, 932701, 548125, 552526, 511636, 386232, 1406216, 471413, 579140, 495399, 1065684, 885308, 791109, 469357, 1276698, 368861, 1011587, 359623, 413174, 1695026, 397374, 417455, 307274, 1236830, 434593, 886202, 597690, 804370, 938497, 1344365, 784315, 1722823, 397832, 943853, 848251, 814910, 332731, 405891, 603827, 354258, 382899, 380274, 374445, 821379, 473860, 677620, 509713, 501362, 516433, 1111792, 1104809, 906000, 1030002, 834502, 1104623, 1226380, 1346974, 1726538, 735901, 395524, 402829, 500227, 291883, 552946, 419407, 482662, 440444, 924185, 1821992, 361449, 773460, 759930, 1139421, 853741, 539789, 391380, 479295, 2150395, 420948, 747680, 546529, 415576, 476867, 385455, 1574757, 416139, 389594, 521960, 1988049, 633242, 584051, 721907, 580062, 1362273, 390253, 552595, 767224, 377215, 584633, 1089725, 1843098, 569364, 527979, 379469, 1045193, 1938603, 767782, 484087, 771947, 336513, 414641, 644849, 691562, 912555, 1083329, 345585, 638112, 1018778, 1122049, 927662, 583028, 464368, 383297, 1021867, 1091663, 383581, 1230719, 1348781, 1152147, 909157, 469025, 358290, 928473, 752556, 424376, 956929, 1483003, 1007997, 358329, 447087, 553617, 844228, 1104975, 1091309, 537845, 409530, 1251436, 1132106, 447579, 905794, 477459, 561825, 897247, 1124120, 1078428, 1447302, 463915, 1533585, 570995, 935819, 727147, 815656, 383484, 1257199, 1335153, 789300, 1047616, 597516, 1041185, 475448, 438910, 382086, 825086, 386656, 1529125, 380178, 400581, 390086], "NumGC":23671} Allocated: 1829391576, active: 1830998016, metadata: 22460528 (n_thp 0), resident: 1903370240, mapped: 1917972480, retained: 4119728128 Mon Sep 13 18:58:24 PDT 2021 RSS = 1785096 2021-09-13T18:58:26.474-07:00 [Info] memstats {"Alloc":95659744, "TotalAlloc":1290683025680, "Sys":283252984, "Lookups":0, "Mallocs":107775119,"Frees":107758139, "HeapAlloc":95659744, "HeapSys":263913472, "HeapIdle":165339136, "HeapInuse":98574336,"HeapReleased":161554432, "HeapObjects":16980,"GCSys":9011200, "LastGC":1631584706470718601,"PauseTotalNs":56606677616, "PauseNs":[447609, 597896, 603321, 1701057, 538014, 455958, 951973, 1408822, 486387, 401328, 338785, 353984, 348737, 1214512, 445842, 711202, 4060860, 461876, 392448, 520905, 390940, 566168, 349086, 569584, 409880, 1107551, 424549, 356875, 456840, 343840, 987807, 658249, 354044, 770576, 2987905, 710864, 675318, 429227, 392839, 378850, 966788, 795475, 1374766, 429150, 899897, 1009908, 1089632, 857541, 919940, 971602, 1085423, 1091061, 970143, 763429, 964435, 428984, 1636026, 632323, 879451, 474713, 699016, 765158, 567336, 909672, 484044, 867262, 611094, 655777, 488538, 1108541, 599676, 495051, 434174, 1413124, 355862, 503896, 1190022, 461683, 992332, 408245, 495506, 421830, 330679, 394471, 370969, 788503, 467154, 846455, 1203792, 374247, 531184, 399986, 307388, 556632, 538428, 390973, 544369, 394766, 1201456, 1385703, 435012, 431663, 545428, 346881, 829465, 679710, 339510, 536361, 414454, 917463, 357646, 467439, 353346, 371600, 825509, 414842, 707483, 1201521, 1161894, 756260, 350373, 1301933, 369276, 1186778, 391027, 1155822, 647322, 1217536, 3975179, 380872, 521136, 495973, 359386, 321682, 381506, 679299, 315649, 410019, 672723, 569976, 740212, 356081, 370723, 472087, 1146451, 808826, 803146, 1092704, 1062917, 424316, 382651, 510647, 1170097, 490707, 853998, 421044, 430217, 634872, 1441335, 1480855, 512042, 1217381, 570956, 824925, 345671, 580800, 724888, 384531, 1242076, 1303424, 1857117, 711463, 577280, 1105575, 1117360, 571747, 618856, 1054726, 988651, 1410020, 883300, 1040838, 581325, 2472286, 584930, 1642120, 523425, 394621, 446726, 739615, 948498, 480874, 1228752, 343693, 1272000, 771711, 867041, 914581, 391706, 515588, 624515, 473556, 847795, 409828, 971281, 405242, 350948, 465743, 378044, 456287, 982423, 1042905], "NumGC":23883} Allocated: 1835250024, active: 1836859392, metadata: 22460528 (n_thp 0), resident: 1899245568, mapped: 1913847808, retained: 4123852800 2021-09-13T18:58:56.521-07:00 [Info] memstats {"Alloc":84645288, "TotalAlloc":1299557459952, "Sys":283252984, "Lookups":0, "Mallocs":108504201,"Frees":108485546, "HeapAlloc":84645288, "HeapSys":263946240, "HeapIdle":176537600, "HeapInuse":87408640,"HeapReleased":162144256, "HeapObjects":18655,"GCSys":8978432, "LastGC":1631584736440181255,"PauseTotalNs":56760851182, "PauseNs":[606702, 687935, 418137, 385189, 1439700, 355168, 504788, 1020591, 488626, 383921, 1437568, 465561, 1069960, 498425, 948355, 959021, 799080, 338359, 517941, 1300359, 597975, 930835, 777078, 485192, 1239421, 556214, 975149, 1029986, 1048112, 523783, 1740078, 659749, 1620045, 966056, 1304320, 1011404, 1423714, 725230, 983987, 479836, 406833, 507869, 354096, 713482, 549411, 330880, 570549, 501275, 805778, 1189436, 418987, 679131, 426837, 552816, 469422, 1212228, 494600, 1265336, 433712, 514372, 602807, 577189, 1208289, 1160016, 770581, 427364, 530372, 1282235, 1327485, 937951, 370756, 615750, 1216524, 509050, 1250733, 551703, 661001, 607915, 337036, 1304918, 738636, 381575, 386434, 1236985, 892702, 880791, 515132, 2354298, 507725, 1273839, 422184, 1019278, 489094, 518246, 496822, 1154312, 569329, 952961, 942076, 639389, 351099, 1321330, 1228744, 1251556, 340252, 442563, 1657503, 1258893, 780118, 421368, 551404, 403469, 494162, 415531, 1677667, 1225347, 437974, 703317, 396544, 361188, 333942, 1488564, 367770, 1524236, 1451155, 949274, 794912, 1169823, 772019, 387622, 420704, 1255634, 422350, 429538, 394821, 794856, 762793, 484484, 1225889, 2043623, 554824, 1231693, 527141, 828861, 488274, 377089, 476851, 346125, 528363, 2068662, 518781, 365328, 644630, 1166369, 695305, 504676, 448032, 1055368, 434966, 371835, 1008723, 508968, 377015, 335811, 351346, 475667, 836254, 518620, 920518, 614102, 1023073, 1129210, 909752, 484434, 340367, 643486, 480827, 359567, 299430, 359424, 373501, 664325, 540292, 939603, 467072, 1102343, 514015, 393820, 774427, 413634, 341213, 517335, 1392572, 497765, 771929, 946869, 840402, 1311765, 540118, 406942, 405751, 459230, 645411, 1387344], "NumGC":24087} Allocated: 1825444896, active: 1827069952, metadata: 22460528 (n_thp 0), resident: 1900941312, mapped: 1915543552, retained: 4122157056 Mon Sep 13 18:59:24 PDT 2021 RSS = 1778484 2021-09-13T18:59:26.574-07:00 [Info] memstats {"Alloc":86475376, "TotalAlloc":1308234980272, "Sys":283252984, "Lookups":0, "Mallocs":109214453,"Frees":109195675, "HeapAlloc":86475376, "HeapSys":263946240, "HeapIdle":174678016, "HeapInuse":89268224,"HeapReleased":163864576, "HeapObjects":18778,"GCSys":8978432, "LastGC":1631584766534931773,"PauseTotalNs":56911911462, "PauseNs":[559157, 367871, 1127364, 373685, 1011421, 1489298, 936016, 482538, 372489, 802914, 633362, 462334, 823426, 539847, 618952, 476360, 1465821, 395806, 441650, 2172938, 514152, 534513, 509300, 1309527, 317829, 464191, 1205824, 1387786, 376908, 511936, 1083714, 2157076, 756231, 450082, 430067, 348111, 537642, 1282235, 417984, 438047, 389441, 872356, 501714, 429663, 557981, 683307, 412118, 1471280, 1434635, 1002555, 433803, 1427381, 570873, 616561, 758871, 616393, 524758, 964415, 1080185, 373565, 546425, 446515, 466393, 468341, 1005895, 429414, 364599, 483676, 392373, 660862, 455663, 453140, 539978, 602043, 1611157, 456587, 556200, 414860, 1037668, 638842, 1272529, 1064066, 1030064, 920006, 374104, 1835434, 348366, 363933, 1137246, 889343, 1019108, 474215, 380113, 819476, 707151, 340593, 477273, 484245, 429427, 730127, 395091, 512791, 490102, 431118, 864821, 1219277, 1014024, 331546, 1034087, 707179, 455586, 879916, 793441, 441475, 522817, 422335, 467866, 401404, 497302, 939041, 463622, 896740, 503059, 518549, 637783, 522153, 462959, 399072, 552814, 1326547, 519806, 1010422, 1005369, 383682, 358966, 388522, 597986, 479756, 1128904, 608515, 426960, 1137891, 1100551, 834010, 1177770, 964742, 772780, 395071, 485459, 429489, 884256, 952481, 494663, 439763, 715055, 1082995, 1846919, 810513, 983157, 1218474, 437734, 347413, 380305, 381998, 429290, 3950669, 1211845, 684904, 383601, 1133302, 934076, 516739, 858148, 981096, 371295, 421301, 627590, 374791, 436342, 1249086, 326143, 346163, 1141705, 425952, 1457171, 828456, 506345, 494596, 1298352, 581672, 490482, 981108, 496954, 1668313, 1117863, 645249, 1098909, 476247, 638575, 750827, 561511, 1035503, 794519, 1210987], "NumGC":24291} Allocated: 1824688096, active: 1826332672, metadata: 22460528 (n_thp 0), resident: 1907281920, mapped: 1921884160, retained: 4115816448 2021-09-13T18:59:56.614-07:00 [Info] memstats {"Alloc":64899384, "TotalAlloc":1316940572600, "Sys":283252984, "Lookups":0, "Mallocs":109932061,"Frees":109915351, "HeapAlloc":64899384, "HeapSys":263847936, "HeapIdle":196067328, "HeapInuse":67780608,"HeapReleased":153583616, "HeapObjects":16710,"GCSys":9076736, "LastGC":1631584796598133983,"PauseTotalNs":57070776093, "PauseNs":[1337903, 1451871, 404977, 940439, 718062, 1428841, 1060523, 1109616, 971666, 555322, 936802, 528315, 1097024, 1129859, 1094138, 434396, 669598, 614874, 1227386, 716059, 913060, 927552, 426906, 1121078, 604577, 882384, 1595894, 1623823, 538810, 547846, 449532, 378251, 2256280, 913222, 476824, 686700, 1018626, 1140557, 589531, 1591095, 465766, 424265, 556149, 672264, 403497, 436455, 466952, 477870, 817301, 1770029, 1095972, 398857, 954856, 1964105, 485320, 468055, 1177780, 1095921, 1206409, 1896039, 428947, 1280533, 646139, 973821, 389247, 456078, 569047, 392814, 506197, 408530, 401267, 571173, 932193, 1188715, 464027, 399454, 888511, 902890, 552475, 987193, 695051, 442032, 1261124, 472493, 396974, 1330627, 473138, 922345, 406908, 578191, 860594, 701011, 481236, 518413, 628839, 706128, 368698, 705040, 974372, 967128, 548756, 827849, 995262, 1578416, 378914, 1509523, 490563, 1424081, 978525, 401544, 1019723, 480442, 602740, 639528, 471823, 470609, 1129291, 406458, 452258, 365992, 639346, 437840, 676294, 469632, 386652, 770953, 857567, 670878, 682374, 364835, 645652, 485705, 395534, 710356, 621373, 408810, 863489, 951668, 475314, 403529, 580065, 884608, 3852025, 1610791, 1049183, 1519376, 998805, 829643, 417642, 464950, 910340, 545364, 1163135, 483767, 842779, 1036665, 444261, 1260979, 1710069, 517138, 430934, 458055, 1091351, 595330, 643420, 363780, 741100, 437860, 948750, 998478, 1254902, 953018, 980806, 335986, 1074062, 510426, 1739040, 1957751, 740709, 407727, 550472, 476949, 1304092, 481976, 461849, 408862, 889497, 701724, 495404, 457110, 523109, 435140, 597709, 718112, 443389, 460760, 1050040, 650723, 382428, 492776, 433668], "NumGC":24492} Allocated: 1832708608, active: 1834340352, metadata: 22460528 (n_thp 0), resident: 1901232128, mapped: 1915834368, retained: 4121866240 Mon Sep 13 19:00:24 PDT 2021 RSS = 1823908 2021-09-13T19:00:26.654-07:00 [Info] memstats {"Alloc":92188040, "TotalAlloc":1325668198176, "Sys":283252984, "Lookups":0, "Mallocs":110653110,"Frees":110635371, "HeapAlloc":92188040, "HeapSys":263913472, "HeapIdle":168878080, "HeapInuse":95035392,"HeapReleased":149266432, "HeapObjects":17739,"GCSys":9011200, "LastGC":1631584826625299481,"PauseTotalNs":57234779440, "PauseNs":[986211, 542694, 2194032, 894429, 420448, 733807, 694961, 524549, 468021, 1049484, 786269, 377894, 1208800, 1082018, 412396, 561227, 404652, 1286084, 409931, 669608, 370894, 445824, 405658, 1251739, 1306214, 1503066, 1406331, 452530, 1360218, 431774, 829700, 734513, 473756, 710870, 782785, 290654, 490083, 482542, 626873, 970346, 1341951, 506760, 329152, 1300859, 550190, 552176, 470200, 778899, 609301, 400345, 1017173, 436289, 362809, 1198688, 1099721, 377661, 403689, 664578, 1157970, 293056, 830742, 1788435, 415777, 1281668, 428772, 1059612, 383289, 387634, 322047, 912654, 845440, 403267, 609216, 873706, 890700, 1328801, 509718, 461965, 438013, 1554583, 1738213, 498221, 438932, 1211773, 814105, 593853, 414078, 474405, 1340707, 830750, 468949, 401749, 1330671, 534335, 566066, 661946, 385655, 810636, 513397, 541515, 931320, 1560883, 1399830, 938868, 1254709, 350209, 12980469, 1114194, 410293, 497620, 752896, 599487, 1324375, 343751, 505683, 1140477, 1083584, 1593951, 1001882, 414722, 535055, 387499, 816360, 426155, 886705, 1161459, 567634, 542962, 818822, 416812, 990499, 1245177, 402774, 400820, 1029507, 519787, 775895, 462734, 584579, 699171, 1515226, 1760604, 1221017, 548455, 370212, 432457, 830852, 621518, 539865, 355995, 450813, 443640, 468892, 809157, 330372, 544145, 399025, 1228522, 554858, 3052757, 408214, 649267, 626990, 470177, 413840, 777225, 487289, 443570, 431234, 455233, 774181, 761224, 682676, 374596, 574341, 611946, 1393883, 1255402, 910927, 578074, 723740, 387969, 382856, 501528, 451393, 529676, 587345, 426358, 802579, 1196662, 412917, 401084, 405124, 617917, 438827, 920319, 1053470, 514755, 965690, 383877, 333435, 1501808, 1454391], "NumGC":24695} Allocated: 1831486416, active: 1833127936, metadata: 22460528 (n_thp 0), resident: 1912356864, mapped: 1926959104, retained: 4110741504 2021-09-13T19:00:56.708-07:00 [Info] memstats {"Alloc":65232536, "TotalAlloc":1334533782480, "Sys":283252984, "Lookups":0, "Mallocs":111381737,"Frees":111364881, "HeapAlloc":65232536, "HeapSys":263913472, "HeapIdle":195788800, "HeapInuse":68124672,"HeapReleased":173260800, "HeapObjects":16856,"GCSys":9011200, "LastGC":1631584856646776893,"PauseTotalNs":57388422995, "PauseNs":[490121, 1296190, 525880, 925038, 290496, 499798, 1278782, 818267, 413360, 327657, 1036108, 659569, 691134, 312750, 675164, 541188, 519780, 755381, 592080, 996277, 787194, 437659, 860026, 365146, 419450, 753665, 1620010, 862645, 1119760, 584547, 560116, 554498, 1126198, 981945, 491933, 1172376, 422158, 422911, 1750594, 1108829, 1205688, 469632, 666189, 986547, 495090, 1254114, 379171, 604884, 967243, 457063, 500200, 1201845, 488180, 464533, 1057150, 1613724, 568405, 497372, 430730, 479849, 1063156, 1052217, 422096, 1176367, 362906, 1272132, 1723704, 417051, 425095, 692478, 530039, 1427586, 444350, 675385, 530728, 1197661, 980228, 1012611, 321640, 2193801, 1757278, 986702, 476689, 980068, 968952, 501172, 580955, 1688121, 925753, 472970, 703347, 894261, 788513, 421354, 485842, 466500, 593303, 1309351, 410238, 482360, 461206, 504850, 960742, 588892, 455991, 590534, 2039022, 491335, 538981, 379181, 359687, 1166222, 606272, 846293, 513339, 498185, 1043184, 543867, 358971, 695244, 419610, 990681, 534768, 970985, 401309, 471817, 1348497, 499989, 434957, 1041337, 781193, 513854, 452796, 1038793, 411801, 1389989, 387486, 396776, 448122, 1021913, 434101, 755733, 627954, 662811, 543739, 1130237, 482074, 487932, 1051185, 500155, 829987, 455103, 456794, 620009, 1450067, 2156168, 1091661, 388285, 332253, 341136, 1380316, 943297, 454226, 345551, 422637, 820087, 689020, 476552, 544059, 466733, 972441, 636782, 357859, 545099, 1452349, 983416, 768526, 478375, 621674, 495213, 521965, 526056, 2202653, 497279, 361843, 927722, 873428, 449242, 499912, 546260, 891458, 2288127, 407172, 1245297, 420496, 449491, 745266, 998431, 962668, 418533, 481623, 389687, 453736, 805025, 380669], "NumGC":24900} Allocated: 1831089912, active: 1832734720, metadata: 22460528 (n_thp 0), resident: 1907474432, mapped: 1922732032, retained: 4114968576 Mon Sep 13 19:01:24 PDT 2021 RSS = 1801656 2021-09-13T19:01:26.759-07:00 [Info] memstats {"Alloc":78335832, "TotalAlloc":1343528896520, "Sys":283252984, "Lookups":0, "Mallocs":112120413,"Frees":112102901, "HeapAlloc":78335832, "HeapSys":263913472, "HeapIdle":182747136, "HeapInuse":81166336,"HeapReleased":156925952, "HeapObjects":17512,"GCSys":9011200, "LastGC":1631584886696491952,"PauseTotalNs":57565632671, "PauseNs":[478474, 836887, 571307, 404224, 521508, 839984, 876985, 403529, 706118, 622971, 374978, 478295, 1131338, 411942, 416897, 1100900, 593973, 513412, 882440, 715117, 376322, 533339, 437754, 1368264, 1253884, 1158469, 429130, 1099278, 1008714, 860073, 1187460, 504000, 1013106, 814429, 854694, 921026, 1662635, 386392, 655129, 451203, 4156298, 684917, 1063504, 1064975, 452221, 490520, 1857702, 1104762, 477936, 411486, 434834, 1053345, 369291, 1104501, 447437, 2121434, 1343705, 490349, 896669, 496908, 1048305, 549140, 1040993, 385396, 1020786, 349353, 391922, 377384, 604490, 950554, 1197351, 959076, 1439835, 477111, 569913, 679421, 953912, 468675, 2768199, 959459, 1166043, 1045597, 414995, 1445315, 1076072, 1371589, 690814, 760060, 476657, 1515433, 490596, 853525, 521152, 1023286, 533346, 1720958, 810584, 1145715, 965692, 486336, 1973602, 794293, 451992, 542203, 392167, 926778, 679472, 753412, 432854, 901053, 473292, 801590, 1221671, 650743, 616280, 517765, 1187162, 1228821, 689691, 675522, 1090941, 381400, 480382, 1134596, 1641557, 922776, 337020, 1253886, 583615, 1019472, 370823, 2023675, 351569, 578881, 508680, 1125216, 1214092, 1138802, 455643, 862271, 391634, 1890868, 1235742, 1216270, 550772, 977207, 413704, 651977, 1354317, 527969, 344729, 380135, 1363933, 415103, 446981, 442085, 1330360, 1476321, 439880, 440361, 381746, 813279, 1114956, 691941, 1012387, 702263, 358448, 444197, 429371, 444566, 1092689, 788549, 506607, 444196, 673463, 988207, 508737, 796773, 1319274, 1130648, 626640, 436543, 515796, 1302928, 575487, 1400200, 376173, 565123, 741409, 943619, 379455, 947985, 329828, 8815518, 833672, 336239, 1172088, 706128, 1208373, 1173845, 580114, 917175, 928683, 993936], "NumGC":25104} Allocated: 1832532984, active: 1834156032, metadata: 22460528 (n_thp 0), resident: 1912352768, mapped: 1926955008, retained: 4110745600 2021-09-13 19:01:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 265, "num_active_sctxs": 92, "num_free_sctxs": 173, "num_trim_sctxs": 19, "num_wctxs": 303, "num_free_wctxs": 241, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 125, "cpu": 8.38000, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1913270272, "memory_jemalloc_allocated": 1835547640, "memory_in_use": 1913270272, "memory_stats_size_total": 1530735056, "memory_stats_size_global": 1187996032, "memory_stats_size_page": 1052296040, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 139840, "reclaim_pending": 4064808, "buf_memused": 338596136, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 16516, "total_ops": 415455, "compacts": 1746051, "purges": 223, "lss_data_size": 260030338, "lss_used_space": 2668271330, "lss_fragmentation": 90, "lss_num_reads": 818101, "lss_read_bs": 128513953355, "lss_blk_read_bs": 125930561536, "rlss_num_reads": 29132, "lss_rdr_reads_bs": 9582538757, "lss_blk_rdr_reads_bs": 3570245632, "bytes_written": 115056504832, "bytes_incoming": 27232244340, "write_amp_avg": 1.46, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 13850, "resident_ratio": 83.85808, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 68533572, "recovery_used_space": 748976518, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86223839232, "recovery_lss_frag": 90, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 660.64000 }2021-09-13 19:01:34 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 19:01:36 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 19:01:36 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 19:01:37 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T19:01:56.792-07:00 [Info] memstats {"Alloc":100586136, "TotalAlloc":1352369300312, "Sys":283252984, "Lookups":0, "Mallocs":112874516,"Frees":112857635, "HeapAlloc":100586136, "HeapSys":263946240, "HeapIdle":160464896, "HeapInuse":103481344,"HeapReleased":158162944, "HeapObjects":16881,"GCSys":8978432, "LastGC":1631584916791343335,"PauseTotalNs":57718913904, "PauseNs":[781030, 539271, 684566, 735566, 464212, 526205, 346637, 824303, 490998, 1344206, 643171, 475362, 367354, 555927, 687322, 523215, 700913, 1230998, 557312, 361187, 1086332, 813395, 432089, 939266, 760728, 386673, 350146, 724379, 438480, 417478, 578612, 561374, 1082483, 1028390, 1262125, 395031, 393008, 521158, 388522, 685144, 1209387, 411461, 350900, 408274, 1017453, 1330520, 777209, 994968, 506465, 342245, 429643, 699515, 558190, 1215112, 429860, 489715, 1411901, 591342, 343495, 822752, 598839, 922777, 1097091, 1124933, 1167211, 339575, 385995, 1609388, 1034524, 2209544, 1267147, 1478049, 619066, 388830, 1096799, 726748, 475955, 720268, 554872, 427972, 2040503, 321115, 422108, 598756, 1239091, 1081215, 619772, 415034, 411887, 345922, 966652, 459872, 1643206, 351199, 518642, 474937, 913300, 498506, 1160785, 544884, 353562, 375163, 382837, 346725, 1793544, 852264, 437486, 342892, 730620, 406579, 1029125, 638605, 392141, 594740, 342632, 443091, 538945, 1047016, 660156, 1029153, 402582, 364874, 877114, 1499885, 524985, 436417, 576750, 408324, 637762, 954566, 873060, 1135295, 382695, 1910143, 502365, 1227854, 752069, 961608, 480899, 330852, 401533, 453381, 988565, 501621, 1049960, 373682, 2173812, 645648, 911696, 1179060, 1276795, 360776, 1422585, 1512532, 932828, 585007, 1003396, 868318, 407002, 1049598, 619996, 823150, 1030388, 414547, 789270, 1310022, 846184, 456443, 471580, 517046, 632308, 1558468, 427366, 674209, 1597729, 633231, 434465, 471677, 411540, 1166826, 500535, 521495, 516170, 371538, 684804, 675521, 457248, 1301441, 762303, 453158, 445139, 475397, 2392418, 823347, 422570, 856118, 713655, 1162083, 382226, 1924980, 676724, 1139254, 563550], "NumGC":25307} Allocated: 1874799304, active: 1876373504, metadata: 22460528 (n_thp 0), resident: 1948393472, mapped: 1962995712, retained: 4074704896 Mon Sep 13 19:02:24 PDT 2021 RSS = 1905376 2021-09-13T19:02:26.839-07:00 [Info] memstats {"Alloc":82262704, "TotalAlloc":1360866684416, "Sys":283252984, "Lookups":0, "Mallocs":113605545,"Frees":113586658, "HeapAlloc":82262704, "HeapSys":264011776, "HeapIdle":178929664, "HeapInuse":85082112,"HeapReleased":164364288, "HeapObjects":18887,"GCSys":8912896, "LastGC":1631584946666108029,"PauseTotalNs":57859504969, "PauseNs":[1554843, 401333, 624812, 536089, 390033, 505662, 1598698, 964591, 414765, 468929, 867759, 954997, 907833, 498292, 448998, 352099, 557987, 903246, 520055, 419403, 708304, 534771, 464287, 1228067, 496581, 1225705, 1098469, 1728816, 324557, 1075730, 1022011, 700751, 444618, 486494, 403831, 750804, 383947, 852007, 345112, 420621, 506502, 958331, 350212, 345641, 1065960, 2045573, 374190, 760259, 543238, 359193, 410256, 1270011, 438533, 368251, 396348, 687885, 407322, 1028312, 951832, 396095, 936026, 435641, 491146, 440057, 477989, 1643435, 1060361, 1144411, 458787, 485848, 1180031, 369451, 639960, 1011483, 1404546, 479747, 871068, 708914, 967121, 538094, 530758, 928456, 603557, 320598, 442528, 519272, 1443265, 497275, 1778877, 361466, 748148, 469670, 575262, 400058, 413273, 1028767, 566968, 589598, 380395, 505332, 421993, 1144065, 574327, 422651, 471869, 498581, 383408, 342690, 440530, 445456, 521136, 558228, 559845, 1231482, 348562, 536878, 1663565, 823291, 663455, 484802, 441417, 396382, 529843, 1283641, 1253838, 439451, 407674, 908597, 424066, 467632, 625510, 1469341, 535569, 869436, 593623, 428551, 1007181, 437574, 385060, 561231, 1032841, 1092420, 433794, 789142, 718130, 1048167, 435560, 340310, 1363929, 923499, 555844, 344586, 471965, 731734, 595363, 930782, 1561441, 372608, 1053996, 1017796, 465900, 569769, 429337, 492836, 1059316, 482725, 682726, 1336750, 453353, 535864, 382398, 455092, 361643, 1659920, 730817, 580615, 2654490, 1203733, 621806, 1028977, 486857, 346366, 722087, 1076413, 587830, 899601, 1252147, 485188, 634121, 832826, 1417725, 552687, 418021, 427211, 475368, 444566, 430156], "NumGC":25504} Allocated: 1914110808, active: 1915695104, metadata: 22460528 (n_thp 0), resident: 1998512128, mapped: 2013114368, retained: 4024586240 2021-09-13T19:02:56.879-07:00 [Info] memstats {"Alloc":66812272, "TotalAlloc":1369446963456, "Sys":283252984, "Lookups":0, "Mallocs":114345896,"Frees":114328752, "HeapAlloc":66812272, "HeapSys":263946240, "HeapIdle":194215936, "HeapInuse":69730304,"HeapReleased":171016192, "HeapObjects":17144,"GCSys":8978432, "LastGC":1631584976831113618,"PauseTotalNs":58019472703, "PauseNs":[943660, 393000, 2266444, 358135, 1205057, 1086405, 882153, 826017, 847555, 1947559, 495161, 358189, 626916, 1030674, 1080445, 984445, 2073556, 968158, 663065, 315570, 397100, 295960, 389834, 497386, 1088148, 1037808, 802790, 392579, 1252327, 1492954, 1017678, 1506756, 553226, 534514, 518925, 655081, 527203, 552383, 781726, 480667, 1133412, 1693942, 1676959, 925126, 799731, 571925, 415041, 1156759, 368471, 985605, 480476, 416730, 365553, 889159, 1006703, 381807, 412533, 1653787, 597015, 490919, 870107, 331026, 437123, 469299, 976856, 402059, 337446, 429848, 445891, 4763977, 1096434, 1165516, 473293, 1221437, 1178345, 396061, 361139, 1614585, 1250533, 430445, 398687, 748660, 798265, 344372, 542157, 504967, 463974, 1188090, 740831, 1018402, 449837, 1872211, 493689, 1873527, 685866, 1345343, 348740, 413831, 1078366, 1233760, 418206, 398270, 1319528, 1383301, 601967, 833149, 896340, 371141, 372718, 381623, 1798003, 1349180, 587491, 370702, 466163, 1379481, 1524649, 1209895, 772356, 537384, 385727, 502439, 441173, 836747, 679225, 628012, 539168, 1228641, 424625, 358318, 410375, 406381, 1145804, 1009811, 1515993, 465216, 586665, 1128326, 498982, 556413, 532588, 1163616, 2033271, 601157, 821976, 990865, 522030, 411027, 652887, 779004, 787540, 1783494, 411358, 1684337, 953280, 438610, 633918, 433034, 509432, 865814, 497853, 1151227, 310536, 1179853, 1062206, 477803, 459777, 997910, 720485, 347383, 1300288, 438432, 492628, 445159, 3580557, 445411, 715091, 725974, 896026, 572534, 499624, 388632, 414363, 402555, 485209, 727767, 363709, 514149, 479857, 665185, 411872, 993197, 598481, 394138, 1106486, 347932, 375162, 509600], "NumGC":25702} Allocated: 1911275120, active: 1912860672, metadata: 22460528 (n_thp 0), resident: 1968037888, mapped: 1983426560, retained: 4054274048 Mon Sep 13 19:03:24 PDT 2021 RSS = 1881700 2021-09-13T19:03:26.914-07:00 [Info] memstats {"Alloc":82528928, "TotalAlloc":1377966479040, "Sys":283252984, "Lookups":0, "Mallocs":115083140,"Frees":115066280, "HeapAlloc":82528928, "HeapSys":263946240, "HeapIdle":178421760, "HeapInuse":85524480,"HeapReleased":170606592, "HeapObjects":16860,"GCSys":8978432, "LastGC":1631585006913265640,"PauseTotalNs":58176167092, "PauseNs":[835107, 664491, 647992, 549752, 1065007, 425664, 419993, 442238, 417842, 820677, 379626, 953205, 1024532, 529723, 505303, 1004730, 1529442, 653238, 572225, 790423, 1023015, 916183, 631881, 531521, 873387, 1381171, 1517480, 400192, 704658, 497459, 876468, 1423123, 408175, 440532, 474404, 1013750, 432590, 471776, 1052635, 1023185, 1024373, 446000, 687239, 366345, 358915, 649552, 437504, 397596, 407889, 1179342, 1675114, 424227, 478097, 407464, 624594, 1031491, 994389, 436580, 1062083, 433780, 1227523, 962808, 1698989, 1057783, 890897, 2105436, 1002591, 1534195, 383173, 634775, 1003310, 432322, 328392, 476073, 523092, 999105, 723921, 1045463, 415094, 512182, 364489, 435631, 1079771, 618790, 645278, 734681, 547678, 1446742, 1146438, 903886, 400526, 453247, 1147133, 910352, 428629, 1316838, 1514057, 457997, 453436, 679724, 481423, 1127439, 452426, 515352, 605906, 1722518, 1562158, 469195, 719580, 540193, 1180806, 505081, 1037504, 364136, 506653, 1320786, 556648, 515340, 1518255, 996756, 482599, 426941, 519331, 591596, 474496, 717410, 1188883, 1076632, 1624658, 383450, 1323490, 554870, 1097301, 1006497, 932138, 518524, 748721, 611621, 868455, 905729, 361495, 356375, 953502, 717167, 1025139, 946802, 520024, 426236, 507877, 463559, 385208, 1755652, 1195642, 998254, 1134565, 1130828, 467913, 720850, 1053360, 1647327, 562702, 499289, 862053, 575851, 376225, 519038, 504041, 1295993, 1261642, 1484632, 1236762, 1821317, 1076621, 375196, 969408, 435027, 413124, 442779, 358063, 490052, 903246, 1014146, 1127301, 1449172, 842977, 762205, 956592, 416434, 490547, 327904, 1016351, 1743987, 483546, 452786, 491594, 668260, 331986, 646452], "NumGC":25900} Allocated: 1922487536, active: 1924079616, metadata: 22460528 (n_thp 0), resident: 2001506304, mapped: 2016894976, retained: 4020805632 2021-09-13T19:03:56.966-07:00 [Info] memstats {"Alloc":68718848, "TotalAlloc":1386258859272, "Sys":283252984, "Lookups":0, "Mallocs":115797622,"Frees":115780720, "HeapAlloc":68718848, "HeapSys":263946240, "HeapIdle":192299008, "HeapInuse":71647232,"HeapReleased":150667264, "HeapObjects":16902,"GCSys":8978432, "LastGC":1631585036938639904,"PauseTotalNs":58321582135, "PauseNs":[1113651, 1019523, 450810, 410797, 515916, 1018865, 1197926, 375105, 445336, 601203, 434393, 543214, 757666, 628579, 737121, 1056153, 415277, 392328, 372216, 450911, 1942534, 353506, 996438, 890560, 1005877, 382219, 860679, 766910, 2432320, 339956, 939483, 399217, 402479, 906263, 1072042, 444043, 575628, 756777, 1522734, 860388, 2162114, 567725, 642686, 367159, 1275245, 1113176, 984225, 480303, 481750, 520748, 1542358, 1499200, 576490, 840965, 627172, 335624, 676541, 464567, 495458, 495197, 955815, 324210, 517357, 1518336, 484183, 1251533, 406186, 536599, 698936, 850598, 642918, 1316413, 499786, 560038, 639764, 411114, 556824, 460986, 366010, 1045687, 406587, 412549, 433163, 464283, 377754, 1140214, 583151, 471124, 449412, 1609292, 710183, 442433, 747252, 1318915, 1649546, 1097884, 516954, 575403, 1333083, 446881, 560807, 1133028, 970522, 438647, 915176, 618419, 1320329, 450962, 1008665, 589821, 867554, 1099199, 1133903, 1155240, 540809, 672642, 431160, 543328, 430810, 400421, 1286564, 448084, 411196, 758059, 1458116, 343119, 664938, 1174712, 384524, 495759, 419753, 703993, 589984, 436559, 1438784, 739684, 589399, 915538, 535457, 329608, 302674, 349299, 377244, 743116, 734861, 420134, 2576960, 486728, 384009, 439620, 428056, 415108, 1000878, 409091, 400854, 402689, 668548, 1151903, 432692, 537202, 387464, 406814, 1243883, 663818, 385435, 404436, 1290200, 792198, 988922, 503668, 552302, 829358, 506505, 1244430, 1111136, 1900541, 446690, 409311, 958063, 656385, 1186120, 942365, 1217946, 463311, 606043, 1276758, 378760, 977848, 1373057, 472553, 417091, 600783, 428288, 1160521], "NumGC":26094} Allocated: 1928744920, active: 1930338304, metadata: 22460528 (n_thp 0), resident: 1996771328, mapped: 2011373568, retained: 4026327040 Mon Sep 13 19:04:24 PDT 2021 RSS = 1902196 2021-09-13T19:04:27.000-07:00 [Info] memstats {"Alloc":64988336, "TotalAlloc":1394909613080, "Sys":283252984, "Lookups":0, "Mallocs":116545660,"Frees":116527589, "HeapAlloc":64988336, "HeapSys":263913472, "HeapIdle":196067328, "HeapInuse":67846144,"HeapReleased":159473664, "HeapObjects":18071,"GCSys":9011200, "LastGC":1631585066902345910,"PauseTotalNs":58471184557, "PauseNs":[477681, 935271, 423521, 503860, 692587, 435747, 467581, 439141, 679595, 1291301, 810546, 566012, 400221, 1187691, 447656, 955821, 1069670, 364914, 462064, 502441, 873270, 1212616, 1031647, 399831, 523955, 449373, 531004, 441211, 539796, 1185123, 765239, 1329567, 2067542, 898625, 756170, 1175742, 782959, 559965, 1221846, 1068002, 977368, 382020, 406320, 649438, 2298763, 320602, 438536, 1504817, 971330, 526780, 468232, 2488397, 349292, 420162, 1358510, 510696, 1685233, 458966, 743153, 419170, 457916, 845292, 1048742, 373016, 561807, 484782, 380629, 1562962, 482440, 491128, 458502, 831708, 1016284, 466833, 892917, 392844, 361721, 486702, 480059, 656640, 494717, 964038, 2031941, 795195, 400102, 765613, 617864, 436933, 1010930, 1537992, 1943118, 467634, 1055532, 654654, 1094976, 398627, 1067964, 452168, 1050350, 708914, 484992, 505535, 345231, 550130, 909675, 1572715, 427349, 581805, 410799, 1492656, 1239102, 980293, 525514, 1064319, 405183, 389329, 692379, 426026, 442975, 765600, 1824741, 1272361, 418514, 480094, 491548, 474547, 1129239, 950500, 789351, 355560, 494331, 825966, 556573, 845353, 853963, 381424, 783829, 1013898, 598957, 1130566, 934937, 364944, 1186068, 577048, 643910, 884232, 517590, 852918, 456952, 462119, 1051379, 913804, 351651, 1412075, 668589, 437008, 752493, 938485, 572791, 410506, 1028051, 766717, 1544690, 417945, 1634514, 324970, 1353586, 1018313, 390962, 399968, 545839, 476671, 556962, 575412, 1004422, 688902, 622775, 487819, 524909, 790768, 614085, 412041, 554908, 565522, 452938, 544464, 398041, 560619, 490110, 1187939, 445179, 1261682, 445922, 529113, 887283, 398550, 530569, 910066], "NumGC":26292} Allocated: 1938469672, active: 1940049920, metadata: 22460528 (n_thp 0), resident: 2008678400, mapped: 2024067072, retained: 4013633536 2021-09-13T19:04:57.048-07:00 [Info] memstats {"Alloc":77978504, "TotalAlloc":1403429383984, "Sys":283252984, "Lookups":0, "Mallocs":117282379,"Frees":117264905, "HeapAlloc":77978504, "HeapSys":263946240, "HeapIdle":183050240, "HeapInuse":80896000,"HeapReleased":170000384, "HeapObjects":17474,"GCSys":8978432, "LastGC":1631585096977509074,"PauseTotalNs":58611217534, "PauseNs":[868450, 774451, 606930, 415893, 572953, 371782, 538116, 454229, 409071, 421145, 1058270, 1198151, 512303, 1058713, 469717, 438186, 678769, 1067864, 442653, 609240, 488292, 750599, 486917, 871752, 446806, 882571, 365898, 389213, 523916, 485647, 827666, 408933, 992274, 495986, 525540, 1481727, 1266703, 358225, 426505, 550219, 1127480, 374303, 360012, 443344, 752315, 763453, 541242, 1087557, 391296, 932030, 452028, 622309, 963236, 1365365, 455438, 411595, 620786, 618774, 1110673, 509859, 1317174, 444270, 586536, 526859, 618993, 927790, 368999, 577693, 1530141, 1121638, 419720, 428313, 1189061, 302101, 1292123, 338725, 568117, 927013, 469930, 423459, 324683, 413523, 1837133, 460946, 636033, 986774, 511790, 383309, 549807, 437923, 1399591, 422841, 438134, 969851, 795986, 652495, 574854, 783606, 2257003, 367095, 1160825, 385493, 998618, 1274389, 582665, 433588, 724734, 452713, 925334, 967978, 719768, 466374, 1387144, 409524, 437907, 583365, 430235, 954256, 556809, 704831, 407600, 565096, 1029129, 1453491, 1044610, 1152429, 1556629, 445024, 1313375, 528251, 440834, 363103, 450567, 480347, 610601, 332763, 926592, 395406, 1111537, 748484, 479440, 1256039, 978438, 953326, 594176, 1083786, 2106158, 1083610, 929043, 795187, 1273751, 596356, 1232533, 533554, 1044783, 452507, 1096155, 469277, 428524, 1009895, 440459, 513905, 419213, 457381, 600098, 653432, 463925, 519958, 1003767, 568968, 1071580, 467686, 425781, 498126, 455547, 394930, 871251, 1127650, 546735, 837708, 393442, 406394, 436300, 428124, 689293, 429227, 400865, 1147651, 422820, 369161, 629217, 318536, 821746, 507226, 628329, 405891, 592730, 393794, 487773], "NumGC":26491} Allocated: 1937418696, active: 1938997248, metadata: 22460528 (n_thp 0), resident: 2048339968, mapped: 2062942208, retained: 3974758400 Mon Sep 13 19:05:24 PDT 2021 RSS = 1885752 2021-09-13T19:05:27.090-07:00 [Info] memstats {"Alloc":76914504, "TotalAlloc":1411784409200, "Sys":283252984, "Lookups":0, "Mallocs":118002535,"Frees":117984217, "HeapAlloc":76914504, "HeapSys":264011776, "HeapIdle":184188928, "HeapInuse":79822848,"HeapReleased":159383552, "HeapObjects":18318,"GCSys":8912896, "LastGC":1631585127002643605,"PauseTotalNs":58773055329, "PauseNs":[1753206, 614653, 415447, 2118559, 512319, 855225, 1423033, 1692146, 1065949, 989405, 1012399, 1384099, 459284, 520811, 569137, 537574, 1258665, 433508, 502531, 896118, 1381919, 559507, 980146, 906797, 756189, 395161, 1083317, 797145, 362871, 372238, 532918, 1016051, 465122, 458200, 1161007, 453137, 702588, 1567669, 959349, 855708, 1156199, 1295110, 1073875, 1146600, 365600, 1231670, 1068444, 1459642, 463073, 1403577, 572215, 710221, 611005, 774956, 1254757, 451478, 910790, 1336908, 1358502, 791927, 515478, 766519, 694027, 448095, 632466, 537484, 689879, 607326, 546198, 1254074, 554822, 753900, 1038437, 374251, 457786, 391811, 513160, 686562, 960411, 477108, 483479, 1707452, 383795, 809593, 1721507, 2076247, 374621, 1411823, 931660, 691934, 1049103, 449858, 1521949, 523975, 536761, 439113, 1232376, 659442, 788769, 418340, 1041864, 741253, 1071340, 407199, 1005264, 369868, 1317558, 1376157, 1487698, 1100705, 2261479, 625711, 714984, 412014, 442193, 1209004, 476749, 519122, 1200280, 1072130, 1112193, 501310, 969894, 1280125, 616343, 475447, 1034834, 538068, 1234386, 1042326, 508819, 385688, 1066056, 468713, 530396, 1418826, 480116, 585416, 988003, 381015, 548445, 458929, 503296, 490667, 490194, 1049027, 549408, 588533, 488174, 1217658, 1073146, 1338864, 530951, 434305, 939926, 1106012, 964641, 518473, 414294, 831382, 602147, 319813, 1762157, 395106, 1444007, 411025, 1099972, 1279738, 500309, 973862, 769507, 882247, 538316, 370576, 682649, 1240992, 553952, 1095743, 819236, 2691417, 569192, 1182411, 589985, 517676, 988095, 1119698, 542185, 504692, 740108, 721010, 1190440, 403414], "NumGC":26683} Allocated: 1932709832, active: 1934278656, metadata: 22460528 (n_thp 0), resident: 2017128448, mapped: 2031730688, retained: 4005969920 2021-09-13T19:05:57.142-07:00 [Info] memstats {"Alloc":79188200, "TotalAlloc":1420296053464, "Sys":283252984, "Lookups":0, "Mallocs":118739695,"Frees":118723106, "HeapAlloc":79188200, "HeapSys":263946240, "HeapIdle":181805056, "HeapInuse":82141184,"HeapReleased":166821888, "HeapObjects":16589,"GCSys":8978432, "LastGC":1631585157134338121,"PauseTotalNs":58930171126, "PauseNs":[459410, 1029312, 388851, 687732, 521932, 1068072, 560821, 429697, 1209088, 579243, 1233523, 938001, 399766, 932758, 460537, 1716075, 386434, 1655604, 1480603, 334524, 422728, 484808, 1579654, 322529, 386605, 863704, 493304, 485139, 487336, 461016, 994729, 443872, 896113, 360329, 766274, 814502, 645162, 363209, 694284, 446160, 537770, 473023, 1255687, 421904, 461081, 445044, 687260, 930309, 491338, 1628962, 403824, 1772191, 1625043, 381874, 1135921, 659477, 1017740, 497766, 559117, 487910, 1261119, 1265214, 517439, 447301, 1213764, 1206066, 1098149, 1144542, 1315035, 693618, 1305001, 460653, 573443, 1381842, 416598, 482035, 737979, 405831, 610126, 427306, 1134958, 620447, 810068, 810375, 906323, 368040, 520179, 987685, 556666, 1233461, 970111, 445795, 635905, 516276, 557442, 953443, 559544, 1233278, 896732, 967080, 823199, 1904325, 716575, 469006, 340130, 613893, 1202975, 558872, 436504, 1586570, 1026942, 554871, 1706986, 1672833, 832348, 563590, 698731, 481673, 475026, 612294, 2136229, 502553, 525142, 1484459, 492153, 581124, 1176806, 468408, 1399105, 586405, 490016, 912848, 633983, 1015943, 653145, 1303514, 424747, 910169, 603340, 962793, 729910, 436056, 777120, 542327, 959095, 520378, 1100255, 489286, 1721572, 633026, 1955222, 472112, 438961, 1292715, 980552, 371107, 1242456, 444707, 496290, 685772, 389778, 535842, 1378415, 739912, 937865, 827004, 641636, 655743, 753559, 489533, 512776, 1177223, 440869, 1001106, 1215637, 1076479, 1387316, 1100250, 475207, 539904, 336401, 424702, 899878, 659196, 891573, 422965, 1278021, 463540, 532850, 802052, 441195, 995438, 1001860, 821579, 1290845, 1344954], "NumGC":26879} Allocated: 1934142552, active: 1935720448, metadata: 22460528 (n_thp 0), resident: 2018701312, mapped: 2034089984, retained: 4003610624 Mon Sep 13 19:06:24 PDT 2021 RSS = 1911676 2021-09-13T19:06:27.172-07:00 [Info] memstats {"Alloc":74216496, "TotalAlloc":1428871325248, "Sys":283252984, "Lookups":0, "Mallocs":119481796,"Frees":119463513, "HeapAlloc":74216496, "HeapSys":264011776, "HeapIdle":186933248, "HeapInuse":77078528,"HeapReleased":159072256, "HeapObjects":18283,"GCSys":8912896, "LastGC":1631585187042927476,"PauseTotalNs":59084118833, "PauseNs":[1065376, 389748, 487611, 836007, 731691, 354422, 790639, 1282238, 531647, 710278, 734511, 490871, 390133, 1097518, 1756825, 480060, 396470, 648702, 1068731, 449837, 527915, 987210, 1081062, 574823, 1001868, 314761, 1042114, 417091, 983366, 661658, 429600, 989435, 492379, 466378, 1847404, 534575, 540302, 1356912, 508059, 488907, 341720, 918482, 595084, 1118583, 514497, 682440, 349249, 617614, 959745, 1159872, 429831, 1191064, 518841, 438944, 534413, 441926, 450438, 492871, 643453, 1401288, 822615, 1167856, 1232959, 405944, 595574, 899133, 1632518, 1120459, 815060, 345417, 636447, 928968, 709421, 552981, 487646, 348107, 457759, 540009, 443821, 458418, 1133247, 543611, 904966, 693230, 436661, 1687725, 718488, 817304, 346292, 2669640, 1057517, 1000905, 355519, 413959, 1441605, 440018, 439795, 366245, 787526, 846173, 1391638, 855474, 1420429, 485064, 494969, 533403, 448490, 934650, 388061, 348242, 1046734, 953382, 639095, 384592, 935252, 521278, 746686, 527902, 599488, 1107908, 534918, 391297, 486960, 1122039, 1063185, 1317043, 1283743, 536233, 1117773, 481269, 434626, 1335670, 626562, 415451, 568636, 1489756, 386123, 1271194, 559816, 417481, 471221, 1386324, 486573, 1052961, 937678, 1401503, 1122963, 815294, 487257, 417524, 900362, 466520, 483740, 747550, 1747547, 702279, 1720839, 1497460, 1062163, 1000093, 960624, 574024, 498126, 544331, 413443, 358077, 325339, 1241546, 854367, 637816, 1208679, 623631, 718380, 1115011, 843481, 1092726, 417683, 577310, 363265, 1569823, 1219733, 631558, 1311893, 1309478, 782231, 3287227, 503249, 399078, 522105, 433009, 723024, 1276170, 542440, 325187, 792253], "NumGC":27074} Allocated: 1937229440, active: 1938796544, metadata: 22460528 (n_thp 0), resident: 2022334464, mapped: 2037018624, retained: 4000681984 2021-09-13 19:06:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 265, "num_active_sctxs": 93, "num_free_sctxs": 172, "num_trim_sctxs": 19, "num_wctxs": 303, "num_free_wctxs": 240, "num_active_wctxs": 63, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 125, "cpu": 8.37167, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 2015207424, "memory_jemalloc_allocated": 1935967872, "memory_in_use": 2015207424, "memory_stats_size_total": 1631297688, "memory_stats_size_global": 1292570224, "memory_stats_size_page": 1156923480, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 139904, "reclaim_pending": 0, "buf_memused": 338596136, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 1, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 18112, "total_ops": 442389, "compacts": 1882070, "purges": 223, "lss_data_size": 260554428, "lss_used_space": 3073304290, "lss_fragmentation": 91, "lss_num_reads": 818101, "lss_read_bs": 128513953355, "lss_blk_read_bs": 125930561536, "rlss_num_reads": 29132, "lss_rdr_reads_bs": 9582538757, "lss_blk_rdr_reads_bs": 3570245632, "bytes_written": 115461537792, "bytes_incoming": 28997714172, "write_amp_avg": 1.35, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 15446, "resident_ratio": 85.28048, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 68664704, "recovery_used_space": 859064710, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86333927424, "recovery_lss_frag": 92, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 724.48000 }2021-09-13T19:06:57.215-07:00 [Info] memstats {"Alloc":90304416, "TotalAlloc":1437344846840, "Sys":283252984, "Lookups":0, "Mallocs":120210827,"Frees":120193794, "HeapAlloc":90304416, "HeapSys":263946240, "HeapIdle":170655744, "HeapInuse":93290496,"HeapReleased":167428096, "HeapObjects":17033,"GCSys":8978432, "LastGC":1631585217205022788,"PauseTotalNs":59241627739, "PauseNs":[1712089, 1955416, 1571633, 577118, 589632, 412341, 603673, 1100390, 523107, 1071492, 391892, 1482413, 511816, 515162, 694807, 1573613, 653410, 522181, 486958, 742183, 851707, 511085, 460206, 522458, 418151, 629692, 972808, 413156, 1474618, 477793, 1073891, 509238, 1218042, 1257109, 1119334, 396002, 413476, 423534, 450894, 392289, 805707, 732234, 1118531, 801553, 728466, 1416295, 696741, 503588, 1293193, 1190677, 950717, 638722, 412591, 610117, 430359, 405455, 887731, 1069463, 390921, 961698, 456853, 340635, 985821, 345650, 889757, 938758, 580833, 778707, 1086425, 595619, 394124, 1108919, 1925780, 502754, 577911, 556658, 865995, 501017, 2010946, 361671, 577704, 446160, 4750651, 584172, 621489, 439558, 611486, 1080869, 1814323, 553263, 1043957, 1062714, 479513, 899039, 489026, 954936, 931861, 332277, 317732, 355128, 530319, 1268553, 670805, 787697, 529544, 482840, 368037, 880122, 511636, 1183204, 1000403, 1063461, 871644, 1211166, 484770, 1322566, 1447422, 470187, 370078, 1300848, 1068073, 843984, 625695, 665338, 523072, 928114, 1251267, 634839, 414428, 756780, 1095128, 353705, 1384785, 376231, 413011, 662124, 413535, 2801588, 1328034, 418195, 475667, 424953, 438696, 1413329, 1570191, 483506, 437780, 979075, 558587, 483209, 1100296, 410878, 898433, 403072, 959853, 460144, 473582, 410891, 449851, 451743, 425227, 411950, 367502, 581646, 356773, 455063, 420065, 1004040, 488672, 511792, 483724, 2046382, 463918, 908995, 462011, 823492, 1712898, 651511, 1061151, 541044, 643913, 403798, 906212, 1516379, 805231, 1230058, 630435, 1442488, 1369212, 436438, 414341, 528076, 534712, 432188, 1667975, 1500371], "NumGC":27270} Allocated: 1946902832, active: 1948479488, metadata: 22460528 (n_thp 0), resident: 2028871680, mapped: 2044260352, retained: 3993440256 Mon Sep 13 19:07:24 PDT 2021 RSS = 1907664 2021-09-13T19:07:27.245-07:00 [Info] memstats {"Alloc":80807752, "TotalAlloc":1446060804896, "Sys":283252984, "Lookups":0, "Mallocs":120962011,"Frees":120943303, "HeapAlloc":80807752, "HeapSys":264011776, "HeapIdle":180338688, "HeapInuse":83673088,"HeapReleased":149258240, "HeapObjects":18708,"GCSys":8912896, "LastGC":1631585247077127340,"PauseTotalNs":59400223337, "PauseNs":[400951, 925363, 933135, 1505729, 1149590, 987788, 1369986, 1122441, 906192, 431400, 537118, 474357, 1595962, 519055, 899316, 1104459, 448680, 2134449, 485080, 1292954, 833099, 421954, 577714, 923029, 773412, 2104437, 1571123, 544954, 699916, 1658936, 505129, 578163, 437192, 704241, 1260806, 1785502, 346762, 464509, 710158, 655929, 509048, 1809758, 1294038, 1164304, 477363, 697370, 509818, 1660145, 429877, 646060, 930063, 657489, 591834, 434266, 440825, 1783842, 652539, 534718, 541475, 443317, 1269187, 1480864, 673791, 1335933, 376481, 1292639, 873335, 482180, 516537, 408517, 1008367, 836741, 773853, 770055, 789096, 925624, 625911, 558166, 436922, 436812, 366532, 671823, 815043, 389909, 879137, 942156, 380555, 601480, 1582575, 510335, 1017175, 586120, 877560, 369186, 1214308, 859675, 906901, 502828, 1290035, 473227, 1782679, 946574, 512272, 702585, 929175, 423223, 439821, 1470789, 1138179, 1016987, 532636, 543804, 542101, 460020, 1097186, 815589, 837731, 1022141, 375476, 1062914, 1274205, 1123030, 873829, 376763, 416930, 397618, 446859, 376588, 428156, 432131, 570945, 721652, 655049, 821974, 1879987, 1013878, 446199, 758247, 706004, 760268, 393094, 1414752, 662702, 374164, 1030060, 1216266, 837934, 449856, 432684, 442285, 558281, 691832, 467816, 516157, 440291, 517989, 1284937, 815460, 898892, 649375, 508436, 911675, 1332228, 383512, 1395340, 388685, 540113, 581709, 556902, 550868, 655581, 559741, 1292791, 784117, 863397, 509464, 541413, 1172043, 553395, 441072, 910901, 1331229, 339153, 347313, 1579908, 1084019, 848504, 1186944, 501469, 548222, 411320, 430031, 464139, 848675, 1157630, 360498, 391798, 1111501, 896151], "NumGC":27469} Allocated: 1942001216, active: 1943564288, metadata: 22460528 (n_thp 0), resident: 2019942400, mapped: 2034544640, retained: 4003155968 2021-09-13T19:07:57.276-07:00 [Info] memstats {"Alloc":63658512, "TotalAlloc":1454342806816, "Sys":283252984, "Lookups":0, "Mallocs":121680527,"Frees":121663140, "HeapAlloc":63658512, "HeapSys":263946240, "HeapIdle":197386240, "HeapInuse":66560000,"HeapReleased":165920768, "HeapObjects":17387,"GCSys":8978432, "LastGC":1631585277164035656,"PauseTotalNs":59553478464, "PauseNs":[643028, 1702965, 754873, 1606293, 1516288, 858554, 427887, 726121, 732933, 405880, 411466, 860220, 1245946, 1089665, 412081, 448995, 374093, 1670584, 1595190, 741330, 1046021, 404936, 709494, 596080, 1116092, 1414972, 424513, 1898095, 402631, 411376, 815272, 484906, 380427, 817179, 360665, 1057334, 677270, 403104, 807285, 445300, 457201, 670871, 921376, 794733, 623335, 555611, 416850, 750803, 1903908, 1423096, 1086136, 523605, 484146, 1776140, 399535, 463019, 577466, 1193053, 443810, 490303, 450618, 486532, 433013, 1441888, 1111701, 886987, 818609, 673585, 1390431, 658102, 822921, 974177, 397428, 414042, 362835, 407826, 1663748, 1160528, 776389, 638764, 658404, 716638, 490490, 1644631, 1710994, 977499, 428309, 553814, 547679, 1210131, 1133759, 1016994, 483635, 1151456, 1053829, 412403, 1063181, 361724, 1548581, 363898, 418962, 956429, 314820, 501853, 449598, 1177754, 339853, 1710526, 435317, 1382646, 400921, 941263, 943747, 901037, 727629, 829489, 530226, 583484, 1808513, 549883, 1119316, 1694273, 343637, 857932, 1335710, 438673, 1121964, 388851, 523366, 418234, 1735534, 702914, 342112, 766924, 730584, 404138, 532549, 945168, 847127, 387817, 358243, 383609, 1203063, 802509, 434032, 548906, 528649, 1235360, 851932, 429279, 475610, 1468616, 395559, 1216060, 686057, 453514, 1143851, 1775795, 384534, 505565, 404833, 1867383, 1946833, 482371, 1277858, 490310, 363590, 428554, 430403, 1140941, 953466, 950524, 569859, 338913, 409637, 498115, 1134026, 903153, 745563, 342675, 441334, 454261, 900459, 447898, 427316, 400097, 863894, 797322, 730668, 376468, 897270, 724309, 942794], "NumGC":27662} Allocated: 1943953544, active: 1945530368, metadata: 22460528 (n_thp 0), resident: 1994661888, mapped: 2009264128, retained: 4028436480 Mon Sep 13 19:08:24 PDT 2021 RSS = 1907664 2021-09-13T19:08:27.305-07:00 [Info] memstats {"Alloc":90302352, "TotalAlloc":1462448748096, "Sys":283252984, "Lookups":0, "Mallocs":122379819,"Frees":122361731, "HeapAlloc":90302352, "HeapSys":263946240, "HeapIdle":170754048, "HeapInuse":93192192,"HeapReleased":158818304, "HeapObjects":18088,"GCSys":8978432, "LastGC":1631585307280947471,"PauseTotalNs":59693336239, "PauseNs":[951305, 1244296, 817555, 887398, 2288549, 468439, 482246, 929006, 550687, 1456287, 1345883, 476959, 695333, 433737, 677545, 1862226, 1200462, 397003, 392676, 814548, 395534, 740879, 304403, 494040, 994946, 500162, 516505, 891457, 428395, 387413, 1838210, 453877, 704072, 1634378, 517816, 712798, 649696, 569862, 413697, 483543, 1558298, 528871, 317175, 826268, 1028656, 1924009, 504209, 388357, 904196, 1045540, 496693, 448305, 531199, 429141, 1039863, 1121568, 593132, 599937, 600625, 486599, 460731, 501109, 858214, 854651, 1030707, 461381, 589892, 510316, 565401, 732152, 1380289, 522182, 443686, 617856, 1171409, 955072, 366252, 367532, 537331, 733960, 731653, 1184469, 1007914, 459577, 1175004, 453990, 1550321, 501402, 632720, 801570, 355915, 817675, 507772, 805325, 477202, 455186, 952300, 489307, 426404, 451261, 508585, 811004, 619994, 376192, 866172, 1132748, 441595, 434253, 343132, 386902, 448929, 480826, 610014, 1131928, 1129360, 1627174, 451067, 438262, 457738, 420918, 452733, 872631, 344512, 609437, 1440788, 957178, 1296761, 574280, 402512, 744464, 1679347, 465699, 322617, 448556, 1358197, 375533, 1107842, 454343, 365734, 986434, 741096, 500951, 741344, 467918, 1657166, 369680, 566357, 982447, 356998, 2147445, 975658, 578300, 302787, 614804, 494489, 552338, 556308, 428172, 1090323, 946528, 695707, 1294808, 453885, 719463, 395698, 672816, 399721, 419787, 1236046, 741377, 1275418, 466600, 554057, 1121077, 976521, 1316307, 1271312, 1026744, 582676, 521412, 380093, 366836, 391648, 441626, 904685, 430320, 626862, 1024723, 428496], "NumGC":27851} Allocated: 1932618800, active: 1934209024, metadata: 22460528 (n_thp 0), resident: 2021564416, mapped: 2036166656, retained: 4001533952 2021-09-13T19:08:57.345-07:00 [Info] memstats {"Alloc":85173592, "TotalAlloc":1470937697960, "Sys":283252984, "Lookups":0, "Mallocs":123114072,"Frees":123094794, "HeapAlloc":85173592, "HeapSys":264011776, "HeapIdle":176013312, "HeapInuse":87998464,"HeapReleased":170262528, "HeapObjects":19278,"GCSys":8912896, "LastGC":1631585337200930252,"PauseTotalNs":59838123198, "PauseNs":[411483, 516135, 590295, 1002647, 453740, 568396, 350626, 1234287, 401844, 429112, 518705, 466396, 935722, 1286653, 592860, 418883, 639226, 1287464, 861577, 376557, 871534, 1005379, 499829, 348092, 1035390, 1037077, 788546, 781702, 974579, 455884, 1369266, 645304, 528791, 529019, 899852, 352907, 899810, 412207, 971532, 797910, 392074, 487370, 530203, 352921, 1277485, 597070, 525774, 533548, 986141, 1059466, 545082, 551298, 435759, 438916, 1339296, 437977, 786132, 439305, 878173, 507923, 828289, 487300, 479662, 367156, 1580998, 412411, 663138, 1054460, 397546, 403917, 385735, 501992, 1135155, 505881, 970944, 1649345, 567740, 1306830, 405629, 347938, 748960, 718335, 449793, 621318, 356588, 892794, 561745, 495919, 1177266, 1014497, 467218, 346615, 510844, 395546, 606916, 1159817, 325718, 430437, 427855, 491584, 377067, 765002, 470605, 363559, 549645, 399287, 1340003, 889747, 379894, 856327, 441667, 607794, 361232, 558983, 456187, 416265, 460612, 1083333, 1232781, 1043315, 1584644, 1158521, 918530, 3937095, 505405, 705482, 723040, 1092636, 1154398, 568171, 1042130, 600155, 1008320, 468136, 1201944, 1459414, 324873, 687386, 700485, 869567, 421290, 1138800, 601378, 2059652, 1217559, 503640, 474529, 431471, 1875230, 325548, 576615, 1245313, 392858, 477057, 576556, 1254122, 404809, 447858, 421807, 493596, 564540, 339300, 801917, 488872, 522052, 2216542, 1108913, 1061043, 731463, 690688, 404902, 538216, 527378, 463433, 471936, 638731, 548719, 1360090, 383745, 500629, 532545, 701217, 1152403, 320243, 1524945, 347385, 484066, 851183, 446790, 1309475, 1178070, 446636, 640402, 780921, 2303282], "NumGC":28046} Allocated: 1938383968, active: 1939955712, metadata: 22460528 (n_thp 0), resident: 2014990336, mapped: 2029592576, retained: 4008108032 Mon Sep 13 19:09:24 PDT 2021 RSS = 1912184 2021-09-13T19:09:27.374-07:00 [Info] memstats {"Alloc":74333072, "TotalAlloc":1479280287888, "Sys":283252984, "Lookups":0, "Mallocs":123833264,"Frees":123815232, "HeapAlloc":74333072, "HeapSys":264011776, "HeapIdle":186810368, "HeapInuse":77201408,"HeapReleased":167174144, "HeapObjects":18032,"GCSys":8912896, "LastGC":1631585367229314022,"PauseTotalNs":59989939161, "PauseNs":[934316, 399730, 1482712, 1096331, 426920, 792997, 3945155, 1700790, 1080785, 401788, 533110, 907690, 526790, 382848, 487785, 689151, 547448, 1056142, 820543, 483093, 598500, 420331, 663302, 492477, 675098, 533765, 1209039, 590293, 835959, 411773, 387695, 700582, 405508, 492256, 845672, 426313, 712307, 742873, 403680, 460217, 761363, 900787, 359909, 1154481, 521999, 743817, 866831, 1944781, 608508, 489818, 1494290, 551347, 380043, 425024, 552112, 790811, 713834, 435293, 681038, 363755, 672661, 478548, 1053210, 1309059, 505581, 665965, 1228039, 545775, 1020217, 1371362, 629363, 1018701, 607659, 1280871, 428444, 405168, 564078, 908952, 439606, 655311, 372055, 537026, 1023951, 453020, 646369, 365912, 531065, 963945, 503019, 707893, 512260, 1344336, 1037517, 476428, 796575, 386680, 449254, 955323, 1117506, 606430, 1532728, 475881, 406487, 595650, 1171311, 1760996, 1906630, 477446, 1931917, 1240408, 1391366, 610266, 479138, 1082425, 413304, 447369, 678043, 499684, 1807946, 1052647, 405968, 928382, 546989, 483943, 935089, 483867, 415941, 422604, 372811, 357264, 592711, 634791, 451210, 530432, 705059, 1137172, 588413, 933116, 517420, 391932, 1097699, 470511, 528430, 449319, 520544, 1288669, 1800858, 973252, 432924, 555182, 640810, 549274, 1278938, 1935652, 937400, 515417, 785340, 934033, 639668, 1277706, 713444, 1209793, 404508, 385540, 799069, 423531, 381977, 392688, 456695, 501176, 462021, 346319, 477694, 460183, 1130154, 1092916, 344272, 823241, 1695788, 767630, 413046, 1098203, 476549, 1833649, 1364799, 511331, 811922, 398424, 1365136, 589144, 903805, 620184, 305572, 674855, 1354216, 992248, 407120], "NumGC":28243} Allocated: 1936676296, active: 1938251776, metadata: 22460528 (n_thp 0), resident: 2012024832, mapped: 2026627072, retained: 4011073536 2021-09-13T19:09:57.420-07:00 [Info] memstats {"Alloc":86266520, "TotalAlloc":1487974738896, "Sys":283252984, "Lookups":0, "Mallocs":124584469,"Frees":124567617, "HeapAlloc":86266520, "HeapSys":263946240, "HeapIdle":174751744, "HeapInuse":89194496,"HeapReleased":163930112, "HeapObjects":16852,"GCSys":8978432, "LastGC":1631585397416337255,"PauseTotalNs":60147162301, "PauseNs":[678489, 487320, 529619, 653963, 787888, 605282, 1219885, 1436942, 556678, 1125913, 339352, 381205, 377642, 628568, 578232, 1809891, 423004, 609549, 939610, 428537, 501525, 457364, 449158, 778904, 399797, 875993, 413565, 1282209, 277869, 665350, 1277577, 471112, 948562, 1093261, 814455, 1037243, 499589, 1390616, 402961, 408257, 740856, 1370581, 386214, 835408, 933559, 1004464, 731231, 396938, 823097, 464983, 380179, 1260425, 669235, 509025, 1644108, 438019, 468907, 1473736, 2479721, 365996, 963941, 558575, 468712, 454069, 1150315, 557788, 567827, 476198, 1029833, 565683, 449810, 371838, 413955, 1725956, 1171232, 1336840, 437074, 811757, 455870, 331131, 422687, 684276, 368099, 629483, 605643, 1635243, 1486427, 1042972, 789988, 1274025, 668600, 699379, 387703, 1299299, 1001857, 937109, 348952, 470967, 508968, 669952, 1220111, 1065240, 875767, 920531, 1031598, 1692232, 481176, 392615, 918660, 819963, 1705705, 1783583, 869515, 1057290, 482249, 902461, 971120, 588661, 903742, 907730, 1188508, 658249, 422389, 538866, 1293312, 1719385, 1168551, 376931, 417292, 486754, 492547, 840400, 1786299, 1587261, 429422, 744549, 874041, 900629, 488131, 724974, 439263, 826387, 1221918, 385315, 1428493, 518693, 585694, 1544283, 418768, 488536, 836435, 785920, 460734, 837280, 1025870, 1037244, 1738745, 1112477, 385782, 770383, 785116, 930270, 583706, 1282071, 416844, 458979, 1179645, 1127636, 353120, 329578, 1303680, 958215, 472610, 894073, 507743, 507794, 439613, 580794, 481935, 942699, 799588, 567338, 367830, 539557, 883171, 391805, 784907, 409659, 1013254, 577819, 947752, 506703, 460352, 438657, 426028, 566911, 694779, 440778, 449089, 728153, 466579], "NumGC":28444} Allocated: 1936579424, active: 1938145280, metadata: 22460528 (n_thp 0), resident: 2002087936, mapped: 2016690176, retained: 4021010432 Mon Sep 13 19:10:24 PDT 2021 RSS = 1887880 2021-09-13T19:10:27.454-07:00 [Info] memstats {"Alloc":72679504, "TotalAlloc":1496627995848, "Sys":283252984, "Lookups":0, "Mallocs":125334087,"Frees":125315684, "HeapAlloc":72679504, "HeapSys":263913472, "HeapIdle":188375040, "HeapInuse":75538432,"HeapReleased":158171136, "HeapObjects":18403,"GCSys":9011200, "LastGC":1631585427378917753,"PauseTotalNs":60299236932, "PauseNs":[979792, 1678337, 648096, 405708, 1141948, 440396, 474214, 500440, 685758, 413493, 726837, 448040, 1110430, 691584, 397418, 1366823, 426668, 883835, 495121, 1055808, 540154, 4567444, 504487, 2134441, 426908, 506481, 381817, 407305, 442281, 1454728, 925840, 504072, 601476, 491419, 611714, 1097976, 581510, 384872, 1202306, 474195, 405334, 1115329, 415564, 562369, 1830216, 512771, 349906, 961660, 509652, 822129, 1216095, 617826, 551798, 1010830, 841015, 639210, 1268386, 747173, 853385, 480330, 1010126, 375507, 431172, 495437, 674810, 1110105, 1215703, 377101, 417074, 667228, 438753, 434921, 356202, 411331, 438571, 404361, 388232, 1416431, 983871, 403430, 1322717, 384775, 477505, 652218, 644938, 646589, 500048, 1681229, 603588, 423672, 1028255, 531089, 721435, 519738, 420108, 373087, 414680, 520625, 795168, 834566, 808409, 392843, 415087, 479102, 402895, 1050390, 1038800, 821290, 928655, 1220976, 418350, 413149, 812205, 576997, 1259775, 376291, 373707, 430827, 377596, 867820, 461521, 908011, 537628, 466813, 457817, 968715, 354501, 673785, 1407966, 322679, 976162, 388282, 460229, 423631, 654224, 874151, 791086, 452367, 480008, 1016447, 1127848, 2061282, 1208528, 481721, 879512, 1207836, 1292381, 725509, 819156, 1171940, 687104, 1348695, 1334441, 464809, 467258, 2788822, 1210097, 587246, 908807, 1120339, 1020781, 1233355, 1021728, 344866, 1002561, 439015, 907414, 748204, 1001431, 360329, 1047253, 612524, 486076, 516956, 782280, 364843, 461786, 467580, 747715, 1426265, 552970, 738174, 369314, 460915, 362196, 443371, 955568, 1798722, 1229987, 513117, 395666, 664579, 1113379, 886232, 1135370, 527367, 436858, 456796, 737024], "NumGC":28643} Allocated: 1941294880, active: 1942863872, metadata: 22460528 (n_thp 0), resident: 2020012032, mapped: 2034614272, retained: 4003086336 2021-09-13T19:10:57.489-07:00 [Info] memstats {"Alloc":62012392, "TotalAlloc":1504721487528, "Sys":283252984, "Lookups":0, "Mallocs":126034357,"Frees":126016689, "HeapAlloc":62012392, "HeapSys":263913472, "HeapIdle":198983680, "HeapInuse":64929792,"HeapReleased":166887424, "HeapObjects":17668,"GCSys":9011200, "LastGC":1631585457474395556,"PauseTotalNs":60445769951, "PauseNs":[520337, 972680, 516825, 428023, 784849, 368964, 809597, 1012016, 661844, 469886, 1119017, 426427, 797460, 857440, 413740, 376995, 688882, 478517, 961842, 1170481, 617560, 474277, 474272, 1547765, 1038516, 427577, 1038494, 500829, 941408, 422294, 760208, 921727, 796722, 475552, 467774, 363425, 371245, 335866, 1230231, 1237619, 366569, 465707, 452677, 1690472, 795117, 1302379, 409290, 754063, 1144707, 754155, 501355, 743834, 391754, 1010538, 482148, 434212, 906729, 1163317, 545715, 544271, 491929, 417330, 823921, 1316748, 1292736, 456612, 405681, 1219326, 467699, 552029, 448714, 938682, 819337, 718981, 1200883, 570707, 397124, 396029, 1398931, 1071781, 456825, 1121297, 569895, 528408, 758216, 632307, 418805, 2779482, 632460, 415112, 529271, 478879, 410628, 887206, 500960, 460145, 698419, 572583, 993333, 1590094, 1303287, 579291, 930849, 848298, 1724841, 366974, 2555229, 484619, 548934, 575794, 906088, 434512, 1010358, 472030, 639099, 386629, 634052, 1175414, 970406, 1359669, 560448, 703234, 464125, 777069, 665922, 449256, 463657, 987913, 365499, 1121138, 2293044, 625899, 1617008, 696024, 563501, 570954, 911347, 974172, 578114, 1310104, 697217, 464112, 432432, 1013144, 948849, 1182881, 973189, 510974, 438058, 1846928, 885530, 517756, 1023384, 1179910, 899781, 402621, 736333, 1014270, 971114, 349763, 585714, 565955, 723000, 367639, 714433, 1867286, 1583555, 643291, 384638, 448684, 416120, 482120, 450161, 556613, 839205, 1111123, 466321, 765518, 438249, 551328, 1356923, 894020, 440773, 514138, 363871, 1052512, 1220753, 611270, 1047062], "NumGC":28832} Allocated: 1963995648, active: 1965572096, metadata: 22460528 (n_thp 0), resident: 2013868032, mapped: 2028470272, retained: 4009230336 Mon Sep 13 19:11:24 PDT 2021 RSS = 1912028 2021-09-13T19:11:27.523-07:00 [Info] memstats {"Alloc":65692448, "TotalAlloc":1513087073960, "Sys":283252984, "Lookups":0, "Mallocs":126757951,"Frees":126740031, "HeapAlloc":65692448, "HeapSys":263946240, "HeapIdle":195379200, "HeapInuse":68567040,"HeapReleased":161095680, "HeapObjects":17920,"GCSys":8978432, "LastGC":1631585487422399089,"PauseTotalNs":60601242371, "PauseNs":[581144, 413079, 511065, 683305, 644199, 542785, 400988, 920749, 373218, 475300, 974661, 623386, 420627, 563280, 374132, 795343, 1133395, 493581, 563926, 582525, 1027124, 984881, 446333, 492612, 431854, 1129393, 820260, 488202, 1480058, 386546, 470113, 931035, 387141, 617078, 341125, 532039, 436636, 1238357, 754206, 485489, 2082730, 758694, 387775, 618499, 595407, 1019596, 1154175, 831852, 459535, 8100367, 1025525, 1136661, 525605, 664909, 385469, 463429, 811220, 1538224, 775199, 520895, 369962, 434209, 342080, 403748, 1566562, 386164, 445467, 773153, 373509, 497770, 1467407, 1782500, 1189022, 367945, 2135435, 440304, 851507, 785002, 1017486, 799111, 501936, 957432, 586341, 422066, 361309, 650636, 351539, 534822, 2057035, 426477, 389937, 1624503, 468872, 423370, 924721, 464189, 591796, 524018, 1014066, 377479, 838463, 411533, 1136078, 311813, 525878, 965253, 822747, 1116475, 600643, 1202620, 304824, 451180, 378895, 495086, 922250, 311233, 446251, 426027, 453933, 420176, 832445, 809100, 320186, 416469, 496301, 1351660, 1413834, 851050, 906176, 452015, 443498, 813216, 456700, 755494, 594983, 1059127, 444740, 1173231, 1555481, 1082561, 1111749, 1780953, 722660, 863004, 628431, 528780, 996643, 1051230, 848140, 365334, 404416, 283200, 761601, 2848581, 475566, 1011474, 439778, 1199629, 1194513, 442854, 1176346, 1857470, 1322792, 711951, 437017, 1523976, 596021, 554144, 521368, 564451, 1165870, 468954, 464673, 684901, 1401079, 1046733, 422983, 779397, 493392, 540883, 354531, 1045557, 884883, 869682, 1448346, 1562727, 411587, 1268251, 464136, 851534, 519762, 1143612, 866590, 430837], "NumGC":29026} Allocated: 1943486000, active: 1945075712, metadata: 22460528 (n_thp 0), resident: 2018209792, mapped: 2033598464, retained: 4004102144 2021-09-13 19:11:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 266, "num_active_sctxs": 93, "num_free_sctxs": 173, "num_trim_sctxs": 19, "num_wctxs": 304, "num_free_wctxs": 241, "num_active_wctxs": 63, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 126, "cpu": 8.39000, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 2032889856, "memory_jemalloc_allocated": 1946287664, "memory_in_use": 2032889856, "memory_stats_size_total": 1640607016, "memory_stats_size_global": 1296372408, "memory_stats_size_page": 1160725664, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 139840, "reclaim_pending": 5441596, "buf_memused": 338661684, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 1, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 18170, "total_ops": 469091, "compacts": 2016752, "purges": 223, "lss_data_size": 260030338, "lss_used_space": 3073304290, "lss_fragmentation": 91, "lss_num_reads": 818101, "lss_read_bs": 128513953355, "lss_blk_read_bs": 125930561536, "rlss_num_reads": 29132, "lss_rdr_reads_bs": 9582538757, "lss_blk_rdr_reads_bs": 3570245632, "bytes_written": 115461537792, "bytes_incoming": 30747976868, "write_amp_avg": 1.23, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 15504, "resident_ratio": 85.32746, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 68533572, "recovery_used_space": 859064710, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86333927424, "recovery_lss_frag": 92, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 726.80000 }2021-09-13 19:11:35 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 19:11:38 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 19:11:38 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 19:11:40 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T19:11:57.553-07:00 [Info] memstats {"Alloc":104982328, "TotalAlloc":1521317340240, "Sys":283252984, "Lookups":0, "Mallocs":127487957,"Frees":127469259, "HeapAlloc":104982328, "HeapSys":263913472, "HeapIdle":156041216, "HeapInuse":107872256,"HeapReleased":145162240, "HeapObjects":18698,"GCSys":8978432, "LastGC":1631585517517200343,"PauseTotalNs":60852344674, "PauseNs":[421844, 527594, 395454, 458620, 473662, 660380, 410015, 546152, 330507, 475707, 1653041, 450516, 518017, 1938568, 420457, 552841, 415369, 756340, 1888418, 482919, 1155882, 927508, 400594, 1915937, 420477, 458794, 513912, 1458748, 2494829, 469313, 955920, 955280, 979839, 4276916, 377087, 381239, 365191, 368675, 1136488, 503956, 442689, 393941, 597180, 558728, 390969, 353443, 482375, 623849, 465756, 718260, 458989, 390896, 1197245, 981541, 1343428, 1308250, 1333416, 572870, 6782818, 4845286, 7129292, 7063787, 2518974, 374206, 660445, 2875370, 4533722, 1016034, 2039546, 5076499, 897409, 1365233, 5621271, 4341158, 1198471, 5120162, 434441, 621829, 521315, 1094532, 636250, 1788182, 796753, 694709, 1169286, 598131, 527209, 632409, 1181902, 462480, 750366, 412305, 450505, 1141305, 637396, 409978, 347213, 372482, 635191, 449543, 486509, 8142071, 444209, 430028, 342229, 1080023, 855336, 4560530, 3900846, 785836, 996004, 584623, 806291, 335266, 4989063, 588247, 1009006, 1018270, 493909, 1036541, 9061619, 6426625, 636799, 815190, 449374, 524975, 487033, 532998, 1592459, 1896628, 4261356, 426795, 512297, 1134908, 841725, 830051, 787594, 1219170, 684758, 605536, 516227, 1189204, 465579, 649328, 950907, 732318, 362860, 523314, 516077, 1187337, 1087779, 737487, 409251, 1393861, 1231377, 818578, 536512, 2275230, 3536560, 663949, 414228, 1350893, 1314668, 4216358, 2023047, 352851, 719358, 667313, 478644, 458886, 1389219, 1212546, 4888578, 451422, 506316, 1059393, 497985, 417749, 483450, 1010451, 426138, 953554, 455465, 531038, 6479639, 4814426], "NumGC":29212} Allocated: 1722547520, active: 1724137472, metadata: 22460528 (n_thp 0), resident: 1785999360, mapped: 1800601600, retained: 4237099008 Mon Sep 13 19:12:25 PDT 2021 RSS = 1810508 2021-09-13T19:12:27.581-07:00 [Info] memstats {"Alloc":100024584, "TotalAlloc":1530076371888, "Sys":283252984, "Lookups":0, "Mallocs":128245711,"Frees":128228286, "HeapAlloc":100024584, "HeapSys":263946240, "HeapIdle":161038336, "HeapInuse":102907904,"HeapReleased":135045120, "HeapObjects":17425,"GCSys":8978432, "LastGC":1631585547567966240,"PauseTotalNs":61091333347, "PauseNs":[604146, 650287, 740649, 1162722, 487482, 1534111, 752965, 780416, 438143, 985498, 1093541, 5580270, 1196078, 1398485, 1046686, 378852, 1239038, 3825978, 876519, 382230, 625680, 431887, 417724, 866285, 1392718, 978510, 814619, 380965, 510234, 513219, 1204767, 570171, 633699, 1930506, 573392, 972441, 654948, 468330, 1491117, 1889677, 3208938, 796655, 1588360, 852336, 1364846, 3091018, 11007993, 959663, 677255, 350252, 726649, 1099155, 1469956, 442733, 1456135, 571624, 668077, 524659, 343643, 1174940, 1718196, 484583, 916981, 1175174, 377524, 346507, 496637, 1551085, 510016, 421532, 763512, 385389, 1047605, 508235, 5391300, 776755, 930532, 795073, 532391, 754787, 459151, 10938909, 951034, 504068, 1228540, 563602, 432283, 4702195, 431578, 350399, 645541, 894696, 997815, 3600433, 791565, 1214406, 1090885, 1246236, 479959, 701231, 3957774, 587309, 384003, 581374, 302623, 581558, 909229, 374979, 438424, 596959, 332735, 686052, 456216, 7681258, 829201, 1107543, 616963, 1196189, 1098802, 5973088, 9893767, 1777436, 497486, 1682598, 1019028, 455672, 8419173, 734981, 608712, 502028, 1080171, 482595, 1523397, 474252, 918803, 432506, 484154, 694688, 714464, 1238049, 358815, 354971, 437429, 403211, 1146865, 1583211, 430373, 694235, 1451347, 552782, 571121, 1275387, 1519065, 1483041, 583611, 904835, 1118571, 429102, 1265525, 1091516, 933216, 417625, 565908, 419658, 982064, 1069786, 1216095, 806126, 772564, 432493, 607319, 611314, 409180, 1251152, 762216, 851502, 589046, 578981, 555626, 756594, 392657, 868247, 389721, 450435, 692625, 989157, 1482592, 3439728, 390267, 1033720, 411065, 647396, 440309, 499220, 4822465, 511140], "NumGC":29408} Allocated: 1829433120, active: 1831026688, metadata: 22460528 (n_thp 0), resident: 1927049216, mapped: 1941651456, retained: 4096049152 2021-09-13T19:12:57.648-07:00 [Info] memstats {"Alloc":70855584, "TotalAlloc":1538336395528, "Sys":283252984, "Lookups":0, "Mallocs":128958508,"Frees":128941791, "HeapAlloc":70855584, "HeapSys":263946240, "HeapIdle":190152704, "HeapInuse":73793536,"HeapReleased":161349632, "HeapObjects":16717,"GCSys":8978432, "LastGC":1631585577601440913,"PauseTotalNs":61261478764, "PauseNs":[419139, 659596, 449640, 915605, 643103, 511902, 1024692, 732623, 1117940, 410115, 342083, 733698, 439357, 572194, 631120, 456496, 633452, 412804, 924510, 902598, 880298, 1031665, 960648, 516191, 1182461, 5545084, 4021598, 1051055, 440471, 1101606, 393120, 1159662, 812054, 433640, 451004, 463637, 1023908, 889508, 481743, 785213, 369612, 1110382, 425742, 998567, 428734, 888288, 3069447, 467451, 734468, 822438, 390407, 499269, 580719, 4395598, 1657071, 2502557, 446868, 548617, 644628, 493279, 1126673, 404167, 359570, 468953, 485499, 594001, 649740, 853198, 386059, 412173, 1388966, 567316, 1023733, 573725, 1008124, 491956, 421025, 477171, 831629, 6348103, 1244396, 369649, 574322, 509141, 433904, 887895, 1362307, 569867, 645113, 1270708, 433855, 431698, 430029, 1216343, 994875, 618046, 519832, 584174, 399876, 977632, 585976, 464112, 645050, 5519005, 387909, 1062283, 545127, 1515446, 833174, 710811, 1509971, 1265636, 1540043, 414139, 835344, 387459, 487126, 353091, 1169998, 580924, 562574, 878335, 699104, 1154367, 384046, 836081, 736593, 1194975, 422837, 576267, 515522, 405280, 345750, 476898, 329653, 361375, 774314, 377628, 375393, 404551, 436975, 332892, 370775, 1103428, 6117118, 1351688, 1034019, 408294, 720470, 640643, 871508, 349929, 454976, 387316, 872653, 557108, 454629, 425623, 530504, 393968, 606242, 485120, 409013, 395378, 374440, 353839, 1118865, 869285, 878818, 510286, 852609, 437981, 544129, 1626189, 532725, 1108444, 379184, 412655, 1150335, 501752, 792026, 1204190, 441561, 455627, 966405, 557341, 1704334, 467668, 993768, 341084, 414192, 6022603, 421828], "NumGC":29601} Allocated: 1884136552, active: 1885732864, metadata: 22460528 (n_thp 0), resident: 1943760896, mapped: 1958363136, retained: 4079337472 Mon Sep 13 19:13:25 PDT 2021 RSS = 1895780 2021-09-13T19:13:27.687-07:00 [Info] memstats {"Alloc":78211744, "TotalAlloc":1546826207936, "Sys":283252984, "Lookups":0, "Mallocs":129693761,"Frees":129675501, "HeapAlloc":78211744, "HeapSys":264011776, "HeapIdle":182943744, "HeapInuse":81068032,"HeapReleased":154836992, "HeapObjects":18260,"GCSys":8912896, "LastGC":1631585607574071697,"PauseTotalNs":61418329706, "PauseNs":[660489, 515428, 745913, 571921, 367129, 497559, 851350, 464397, 542926, 516892, 742293, 652458, 490271, 2673581, 405381, 1384466, 1256378, 1933978, 375476, 544828, 914968, 1241726, 1883196, 439998, 2661209, 593222, 561582, 501133, 396779, 591530, 441754, 1035904, 941803, 1061900, 369538, 986078, 403641, 463307, 389271, 371654, 437163, 1147820, 696801, 995922, 1929233, 363442, 389438, 972114, 389538, 389449, 714808, 447950, 1025994, 438985, 425625, 470410, 742703, 354978, 1402945, 1278462, 791800, 1399128, 400946, 472480, 945942, 367999, 1868422, 1383382, 586574, 441884, 775237, 1305259, 541825, 876290, 499410, 1084082, 1170518, 1144705, 590130, 1085030, 562647, 446879, 1173497, 670050, 548598, 604774, 1489580, 1664445, 614382, 1209607, 598027, 2121912, 605234, 675038, 451157, 633599, 1487841, 789585, 511251, 539869, 759717, 992339, 954029, 524862, 992494, 379543, 775140, 798075, 361048, 460724, 415354, 998984, 553676, 1558926, 1003335, 1011304, 530005, 569621, 407367, 788184, 1472333, 628091, 4598285, 603757, 450582, 382825, 555817, 645978, 785905, 1016128, 1193417, 1194837, 433175, 946292, 656773, 366755, 507768, 985258, 762056, 548584, 1284012, 432828, 571614, 499871, 406727, 443029, 810484, 962224, 411710, 399169, 405900, 400586, 394380, 502047, 1109343, 379014, 361109, 1169008, 714544, 1143238, 1022669, 632830, 421165, 361012, 875353, 402930, 511682, 452851, 1073110, 650007, 926407, 811852, 392414, 423156, 1136531, 1451964, 1092590, 1133865, 2376701, 834732, 925586, 1277124, 1140910, 907663, 391781, 742377, 425919, 1413341, 368518, 690397, 1243349, 667458, 716416], "NumGC":29794} Allocated: 1918885656, active: 1920471040, metadata: 22460528 (n_thp 0), resident: 1997963264, mapped: 2012565504, retained: 4025135104 2021-09-13T19:13:57.727-07:00 [Info] memstats {"Alloc":85683216, "TotalAlloc":1555465691120, "Sys":283252984, "Lookups":0, "Mallocs":130439775,"Frees":130420594, "HeapAlloc":85683216, "HeapSys":264011776, "HeapIdle":175497216, "HeapInuse":88514560,"HeapReleased":167665664, "HeapObjects":19181,"GCSys":8912896, "LastGC":1631585637569453241,"PauseTotalNs":61575923462, "PauseNs":[282647, 420120, 389859, 499619, 1077709, 413454, 1000940, 1169877, 473220, 1055640, 1194516, 545912, 468833, 1174684, 384618, 1697148, 335366, 594050, 398077, 372065, 1230896, 629587, 1371449, 383022, 418786, 463358, 940000, 869536, 396239, 354876, 655067, 2016516, 770905, 601619, 451713, 773603, 1228093, 1761195, 490179, 5746705, 380859, 432294, 4247202, 422761, 922523, 1210710, 1095370, 524506, 997075, 1409244, 435097, 366933, 406160, 442728, 498832, 435747, 1148777, 1043071, 462737, 340722, 385155, 613841, 941406, 1176453, 674579, 521487, 1112521, 1142520, 1139217, 981141, 1113958, 982783, 459663, 502779, 1070179, 653742, 459931, 1098679, 460158, 463083, 391944, 401115, 989005, 620462, 2021584, 461457, 2329043, 2199032, 1092245, 613203, 473007, 516538, 429696, 427570, 461299, 421016, 551602, 1027640, 589288, 874976, 631419, 1329541, 752689, 545047, 836528, 864198, 946572, 327715, 1713701, 384382, 443772, 883241, 442700, 1199938, 558330, 720680, 667091, 1041374, 883364, 645042, 1094107, 934018, 441049, 404351, 426372, 604348, 1232800, 510722, 617233, 416914, 814439, 425172, 471419, 870897, 1239706, 616298, 534954, 472020, 891201, 941026, 1101646, 761512, 354865, 403539, 317116, 1165650, 574753, 648170, 728468, 538864, 399809, 1261901, 806744, 596604, 1348240, 451943, 628340, 369241, 675032, 1116984, 537751, 795384, 969779, 450700, 1576262, 707355, 1103901, 413713, 473939, 1153694, 1221464, 524248, 368219, 494845, 763772, 636784, 511416, 437297, 724573, 665909, 444713, 432893, 444942, 1188392, 627051, 347443, 434431, 385519, 587558, 542015, 564308, 1210329, 386815, 516605, 708624, 444500, 378183, 975755, 354215, 468350, 696512, 1327764], "NumGC":29996} Allocated: 1932138872, active: 1933725696, metadata: 22460528 (n_thp 0), resident: 2024620032, mapped: 2039222272, retained: 3998478336 Mon Sep 13 19:14:25 PDT 2021 RSS = 1915688 2021-09-13T19:14:27.788-07:00 [Info] memstats {"Alloc":94385800, "TotalAlloc":1563870580600, "Sys":283252984, "Lookups":0, "Mallocs":131165200,"Frees":131147514, "HeapAlloc":94385800, "HeapSys":263913472, "HeapIdle":166535168, "HeapInuse":97378304,"HeapReleased":157073408, "HeapObjects":17686,"GCSys":9011200, "LastGC":1631585667783370908,"PauseTotalNs":61742054717, "PauseNs":[904167, 518790, 438193, 452079, 452687, 484545, 475965, 514973, 510011, 14336255, 491363, 541973, 355425, 653817, 1220583, 587611, 1150890, 480862, 468881, 466246, 563805, 1134791, 1048859, 842869, 1037729, 547399, 534750, 496939, 420791, 478628, 466747, 516049, 1234933, 378763, 343236, 477833, 523042, 565034, 1829134, 476223, 889964, 993737, 423705, 946631, 1254837, 1597862, 861546, 726743, 696754, 506167, 433894, 450446, 1653319, 497921, 433124, 448072, 1026677, 808166, 1671131, 1554599, 502695, 375138, 839624, 522107, 1287747, 624960, 814462, 633610, 475639, 933719, 1515089, 455020, 520070, 1010427, 1168459, 544881, 384135, 810166, 1106678, 952532, 430277, 378139, 446104, 794198, 413361, 1330968, 1022284, 379312, 385081, 1461521, 829181, 2026607, 347423, 980352, 856391, 736537, 651936, 1134158, 605430, 495971, 592976, 707419, 419012, 431254, 473364, 570877, 465013, 593274, 599837, 1636323, 444287, 532784, 439112, 855508, 497580, 999404, 1251723, 593301, 359203, 354212, 1015740, 2222026, 1081647, 539778, 1129251, 425949, 463214, 661002, 933080, 1874930, 720691, 681631, 1580995, 693084, 551863, 1261735, 439601, 523020, 919960, 540411, 433359, 1343816, 386841, 1256812, 335823, 947103, 1354351, 1399232, 421738, 787240, 1068567, 1013567, 524746, 452367, 463493, 1483534, 797732, 968724, 947959, 869408, 1309146, 872814, 1390977, 343403, 1405897, 1010258, 581930, 551185, 394948, 393978, 452262, 949321, 632916, 1092446, 1343517, 421129, 429806, 1176020, 490738, 884616, 397907, 964875, 439440, 608493, 959373, 574954, 1220426, 902824, 423024, 1038442, 330542, 835051, 745172, 1179445, 466761, 440143, 1664039], "NumGC":30193} Allocated: 1962527232, active: 1964118016, metadata: 22460528 (n_thp 0), resident: 2020818944, mapped: 2035421184, retained: 4002279424 2021-09-13T19:14:57.836-07:00 [Info] memstats {"Alloc":67206280, "TotalAlloc":1572165552112, "Sys":283252984, "Lookups":0, "Mallocs":131883303,"Frees":131865934, "HeapAlloc":67206280, "HeapSys":264011776, "HeapIdle":193830912, "HeapInuse":70180864,"HeapReleased":164995072, "HeapObjects":17369,"GCSys":8912896, "LastGC":1631585697722178394,"PauseTotalNs":61899129434, "PauseNs":[1036592, 405540, 518504, 1411662, 381734, 489340, 890001, 793768, 1033013, 878821, 405737, 1225428, 560791, 1146548, 654542, 968549, 538952, 543974, 649723, 403978, 432750, 509805, 730206, 529985, 400142, 1232279, 453704, 1223739, 618399, 483976, 1009072, 334852, 408819, 1565611, 850978, 446000, 454907, 419457, 416519, 919191, 1044593, 1489382, 996740, 405198, 555914, 558047, 428461, 1237809, 570261, 1132330, 415441, 653232, 391508, 445510, 498280, 1045373, 426901, 2009098, 532905, 696135, 469284, 536411, 493673, 705111, 400625, 1093306, 567035, 480927, 1091267, 710041, 495601, 1071332, 826851, 3399721, 414251, 489609, 715298, 876745, 1374238, 1660310, 381543, 560491, 463284, 810238, 898662, 381951, 442050, 1220709, 401126, 378820, 373392, 701260, 1200007, 988483, 1227668, 322594, 483993, 926725, 790767, 899109, 750746, 798555, 343211, 465296, 390985, 1618080, 1022894, 1117425, 455156, 396767, 616261, 529405, 659682, 1885296, 392624, 1092549, 452786, 465893, 611988, 766745, 435474, 795879, 397218, 376484, 397255, 2410240, 856430, 675595, 1836400, 766275, 1031047, 601241, 610206, 1430299, 463356, 405750, 356648, 919717, 1116486, 1352793, 500980, 1420884, 506589, 1402906, 424545, 360956, 1002834, 449504, 1540443, 7908831, 462175, 442702, 1201627, 607619, 1299070, 1582328, 1296646, 437070, 419178, 424370, 540053, 1009140, 652203, 938388, 1224393, 1191273, 458932, 873557, 738999, 1133561, 715794, 341646, 487381, 1346898, 530335, 688874, 1501930, 1025773, 512009, 525094, 439205, 447128, 838341, 1622492, 614151, 340166, 369968, 987740, 415674, 1318975, 802362, 561129, 487545], "NumGC":30386} Allocated: 1941276656, active: 1942867968, metadata: 22460528 (n_thp 0), resident: 2019573760, mapped: 2034176000, retained: 4003524608 Mon Sep 13 19:15:25 PDT 2021 RSS = 1899808 2021-09-13T19:15:27.885-07:00 [Info] memstats {"Alloc":90403896, "TotalAlloc":1580721208160, "Sys":283252984, "Lookups":0, "Mallocs":132623542,"Frees":132606933, "HeapAlloc":90403896, "HeapSys":263946240, "HeapIdle":170573824, "HeapInuse":93372416,"HeapReleased":158326784, "HeapObjects":16609,"GCSys":8978432, "LastGC":1631585727884296056,"PauseTotalNs":62059279121, "PauseNs":[496810, 1432916, 511534, 499026, 396845, 367023, 1425636, 452129, 1557942, 1847550, 429069, 437370, 1888445, 1689123, 692630, 1234892, 485442, 771352, 1612697, 623979, 396068, 472163, 689884, 1011568, 1098573, 981456, 656109, 1229709, 1283045, 999155, 776295, 1186022, 888376, 1213640, 623700, 385342, 1132258, 404871, 457032, 536024, 693437, 497578, 484442, 604407, 740270, 817427, 1133616, 1789100, 870029, 1100446, 380898, 1531314, 528252, 451286, 1440150, 772892, 525620, 399000, 1036600, 663782, 568633, 634705, 426135, 1394489, 344052, 549224, 466773, 692412, 468436, 863113, 734630, 586796, 1130052, 688838, 346818, 523508, 1209427, 1429928, 366977, 476236, 561788, 398861, 938194, 444290, 672126, 1017555, 663140, 433451, 511951, 585789, 1722166, 845722, 1719338, 1000994, 1558226, 1137522, 484299, 460559, 739089, 1717458, 884223, 1038464, 454217, 624450, 755760, 842412, 1137172, 773196, 560147, 866725, 842847, 1335172, 512194, 1210736, 386467, 1381059, 348026, 375581, 6298820, 523949, 512468, 849762, 1071544, 534072, 717253, 467236, 414680, 457863, 1450151, 618996, 737317, 540263, 479330, 901829, 1042907, 329492, 417131, 382817, 969834, 473317, 514123, 447034, 395500, 1161790, 371017, 921892, 393775, 433698, 519130, 397652, 421557, 468825, 699018, 943432, 1029178, 642420, 1693599, 773772, 435351, 961905, 690124, 1063973, 686617, 1068249, 357816, 1128369, 384738, 413697, 420776, 417952, 397379, 1191519, 488550, 406337, 959956, 1687436, 496426, 431487, 347769, 385263, 332590, 396737, 1032910, 580739, 1292881, 759004, 812184, 430870, 1534719, 548651, 1914264, 473105, 806346, 1240757, 390002, 342112, 367648, 1365083, 712159], "NumGC":30585} Allocated: 1946600304, active: 1948192768, metadata: 22460528 (n_thp 0), resident: 2066186240, mapped: 2080788480, retained: 3956912128 2021-09-13T19:15:57.915-07:00 [Info] memstats {"Alloc":84053872, "TotalAlloc":1589296653512, "Sys":283252984, "Lookups":0, "Mallocs":133366184,"Frees":133347012, "HeapAlloc":84053872, "HeapSys":263946240, "HeapIdle":177004544, "HeapInuse":86941696,"HeapReleased":150986752, "HeapObjects":19172,"GCSys":8978432, "LastGC":1631585757881386849,"PauseTotalNs":62219197538, "PauseNs":[1312464, 1268357, 815950, 866506, 1158210, 1116892, 478481, 452997, 558339, 414988, 432163, 465686, 1009383, 390898, 478978, 399808, 647858, 904176, 461368, 954966, 1223667, 371909, 461892, 797671, 1125996, 753526, 862101, 403337, 1244746, 511156, 890467, 1060303, 500251, 1062706, 818798, 393164, 1089671, 618915, 544137, 1489710, 541745, 1261608, 470253, 384423, 1003142, 834552, 1137119, 809129, 365772, 440116, 1266105, 496646, 923025, 639148, 1074476, 705447, 1200299, 445610, 436699, 1064566, 977126, 1016595, 1174530, 555850, 1412908, 881549, 378478, 399842, 429795, 1104260, 1366689, 616050, 1421592, 416480, 1162251, 1527904, 705804, 1315424, 600977, 421854, 1104173, 552641, 1730665, 672115, 902950, 524592, 735847, 661144, 712783, 790819, 1188063, 760117, 574923, 1158787, 423908, 448860, 1193123, 475016, 1577803, 1645353, 402144, 874807, 433020, 778595, 895974, 971595, 1552358, 1219662, 1014941, 402228, 795253, 920777, 892004, 416245, 1815743, 906476, 899756, 435773, 523399, 797898, 1202810, 632979, 678202, 592487, 392342, 580403, 493959, 393405, 448743, 396578, 966135, 1156847, 371288, 464140, 497043, 396944, 496292, 939355, 432794, 444070, 390474, 369639, 1333618, 494335, 1075224, 1764778, 426803, 425351, 558274, 410841, 626946, 465473, 338969, 409940, 1017089, 1564671, 467197, 446117, 396721, 363385, 1073297, 477035, 1178616, 1328534, 743614, 682941, 947908, 782348, 1464477, 746113, 913461, 410883, 634274, 923955, 1018543, 974568, 433084, 846317, 866058, 925621, 922451, 533145, 533670, 1593425, 536741, 575237, 1145681, 1778802, 1091171, 446535, 509267, 895339, 500639, 781633, 929978, 1046613, 2659508, 1763515], "NumGC":30783} Allocated: 1936180392, active: 1937772544, metadata: 22460528 (n_thp 0), resident: 2021457920, mapped: 2036060160, retained: 4001640448 Mon Sep 13 19:16:25 PDT 2021 RSS = 1919292 2021-09-13 19:16:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 266, "num_active_sctxs": 97, "num_free_sctxs": 169, "num_trim_sctxs": 19, "num_wctxs": 304, "num_free_wctxs": 233, "num_active_wctxs": 71, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 126, "cpu": 12.56149, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 2027962368, "memory_jemalloc_allocated": 1955710616, "memory_in_use": 2029010944, "memory_stats_size_total": 1646435780, "memory_stats_size_global": 1291203464, "memory_stats_size_page": 1155481200, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 139856, "reclaim_pending": 7342800, "buf_memused": 347833708, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 15, "num_active_smr": 7, "num_pending_smr": 2, "num_pages": 2080, "items_count": 25, "total_records": 18112, "total_ops": 496023, "compacts": 2152097, "purges": 223, "lss_data_size": 261220116, "lss_used_space": 3539027682, "lss_fragmentation": 92, "lss_num_reads": 819928, "lss_read_bs": 129284715326, "lss_blk_read_bs": 126189293568, "rlss_num_reads": 30959, "lss_rdr_reads_bs": 10353300728, "lss_blk_rdr_reads_bs": 3828977664, "bytes_written": 115927261184, "bytes_incoming": 32513315604, "write_amp_avg": 1.16, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 15424, "resident_ratio": 85.15901, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 68795836, "recovery_used_space": 969021830, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86443884544, "recovery_lss_frag": 92, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 724.48000 }2021-09-13T19:16:27.962-07:00 [Info] memstats {"Alloc":81128600, "TotalAlloc":1597830288448, "Sys":283252984, "Lookups":0, "Mallocs":134101220,"Frees":134082309, "HeapAlloc":81128600, "HeapSys":264011776, "HeapIdle":179896320, "HeapInuse":84115456,"HeapReleased":158449664, "HeapObjects":18911,"GCSys":8912896, "LastGC":1631585787893071077,"PauseTotalNs":62382421583, "PauseNs":[401487, 869906, 574803, 490822, 1073487, 1185851, 400282, 806021, 390951, 522274, 554174, 1794884, 552336, 416864, 1077100, 1573592, 838110, 971621, 337601, 1169188, 1109764, 408872, 689535, 462060, 1120042, 686054, 367999, 608191, 1016061, 491636, 940966, 352406, 396569, 442700, 1230090, 1511118, 1629015, 403624, 475722, 454218, 820271, 1397707, 2141642, 1003859, 1033875, 690862, 552564, 1326694, 678508, 330540, 2080394, 940543, 419022, 642764, 568853, 427855, 399941, 981268, 479045, 410808, 1276551, 1597075, 348360, 511699, 888377, 578167, 482309, 505837, 1103805, 1277651, 442297, 458274, 1697727, 821744, 726103, 1215015, 1423922, 420457, 1296488, 740372, 1282539, 319533, 373063, 467002, 1451391, 1567059, 599354, 669635, 546004, 555507, 1386914, 400794, 526262, 827526, 360862, 540791, 930301, 446516, 532424, 358601, 1353702, 1161894, 1116810, 860752, 465596, 442378, 743307, 1091666, 2953050, 1676744, 917411, 549782, 448763, 571846, 586576, 609374, 2002435, 548578, 1139588, 510828, 368496, 418267, 371698, 1013478, 659510, 492328, 488232, 688650, 473647, 1979440, 630454, 7150159, 796473, 772029, 727671, 1034188, 391584, 512629, 1318122, 359612, 785800, 378727, 1125545, 950910, 657906, 361365, 1098284, 643355, 870215, 484096, 443030, 925670, 474158, 602908, 438731, 590596, 414400, 514660, 500119, 436775, 649089, 552897, 1812916, 450181, 569337, 400661, 589103, 460824, 452044, 513609, 349617, 522179, 1564239, 793551, 823860, 354786, 327609, 399198, 1123148, 355702, 824635, 1447518, 1457068, 933517, 543763, 863325, 533564, 1029391, 801360, 1253641, 1331722, 701837, 1003054, 528013, 424902, 1040101, 1032300, 1235468], "NumGC":30981} Allocated: 1948010136, active: 1949601792, metadata: 22460528 (n_thp 0), resident: 2028748800, mapped: 2043351040, retained: 3994349568 2021-09-13T19:16:57.989-07:00 [Info] memstats {"Alloc":81684504, "TotalAlloc":1605820515248, "Sys":283252984, "Lookups":0, "Mallocs":134792925,"Frees":134774905, "HeapAlloc":81684504, "HeapSys":263946240, "HeapIdle":179355648, "HeapInuse":84590592,"HeapReleased":151142400, "HeapObjects":18020,"GCSys":8978432, "LastGC":1631585817896393149,"PauseTotalNs":62520709925, "PauseNs":[959969, 357875, 467791, 505590, 432578, 427723, 724956, 453932, 549361, 326899, 848658, 991585, 1278295, 715437, 423686, 1557733, 437620, 444197, 1269425, 664079, 689157, 580292, 473632, 561375, 444668, 454094, 445317, 487474, 555924, 375536, 871748, 1219733, 1051746, 998937, 767613, 569185, 1791154, 516378, 471799, 5143882, 478529, 533405, 511336, 1717017, 420251, 737990, 476066, 740573, 639619, 1054596, 485875, 623866, 373881, 916490, 431305, 457534, 573877, 353416, 1180933, 472016, 655332, 452855, 429602, 1337900, 376543, 3073242, 442017, 439810, 936623, 396547, 801688, 632844, 801544, 872283, 407811, 1480371, 610358, 436399, 1011369, 974431, 368799, 456453, 454941, 1047596, 1385662, 825604, 419768, 1715031, 605000, 536081, 1030495, 931156, 504363, 551708, 386262, 1429651, 516785, 764372, 567435, 391456, 379831, 492471, 887086, 779430, 1206595, 1694655, 404888, 788500, 402383, 797099, 681316, 471776, 404843, 722851, 424052, 1081302, 1184780, 440931, 521166, 470878, 456465, 528741, 1039678, 668295, 612169, 1023997, 533833, 315389, 1222219, 1209942, 388255, 404468, 489542, 1061116, 520575, 562768, 476789, 488614, 569028, 416767, 508427, 992468, 1512334, 530155, 372224, 426767, 444366, 438279, 1235637, 573304, 531857, 615829, 949559, 1616985, 444466, 327036, 421064, 427871, 1840948, 678407, 477954, 445410, 506712, 680771, 1019637, 1256396, 1703300, 402631, 1041194, 478974, 500956, 783260, 1390895, 438709, 643356, 478225, 521166, 1150941, 440803, 997571, 527847, 428681, 559578, 383189, 1091437, 495263, 386225], "NumGC":31168} Allocated: 1946928400, active: 1948508160, metadata: 22460528 (n_thp 0), resident: 2049839104, mapped: 2064441344, retained: 3973259264 Mon Sep 13 19:17:25 PDT 2021 RSS = 1896900 2021-09-13T19:17:28.035-07:00 [Info] memstats {"Alloc":72948088, "TotalAlloc":1614255509792, "Sys":283252984, "Lookups":0, "Mallocs":135521269,"Frees":135502771, "HeapAlloc":72948088, "HeapSys":264011776, "HeapIdle":188170240, "HeapInuse":75841536,"HeapReleased":158449664, "HeapObjects":18498,"GCSys":8912896, "LastGC":1631585847885873279,"PauseTotalNs":62679306149, "PauseNs":[664828, 395952, 436940, 493668, 1163434, 644361, 442221, 687844, 632953, 446927, 511861, 489685, 415904, 908554, 1315041, 1114609, 456980, 386022, 984454, 565738, 1143604, 477199, 1088868, 441994, 462904, 858937, 2170739, 364983, 345452, 441747, 359292, 305278, 636959, 429844, 575348, 374858, 489748, 530797, 772949, 584674, 1112678, 466476, 887851, 1104684, 378721, 503619, 1271516, 533309, 987717, 505806, 2073911, 761126, 388130, 559660, 1250949, 485394, 609609, 494364, 321103, 2215471, 1807273, 390445, 737001, 1097345, 391375, 526311, 441253, 577392, 348719, 402204, 348219, 399061, 471872, 298593, 1043947, 1015784, 534179, 389263, 374349, 635939, 649096, 2244325, 626679, 1455947, 512385, 417945, 360841, 532073, 1057922, 699789, 564160, 670246, 2080447, 1178185, 351896, 659025, 914882, 603867, 512746, 1908294, 6600702, 1094465, 502578, 388191, 499589, 395943, 571427, 361279, 414947, 653308, 554911, 395580, 676578, 1066176, 812375, 559008, 923408, 743988, 557421, 304821, 910572, 456082, 1477659, 540744, 472267, 443753, 471841, 818283, 452013, 999284, 475139, 1006175, 354264, 3295016, 471477, 368550, 837893, 1372405, 1303955, 423234, 568797, 549537, 438699, 911412, 550188, 1535825, 376868, 889548, 920018, 400948, 1022295, 632483, 973314, 533011, 443353, 452806, 335559, 1397748, 507685, 439959, 446703, 1026304, 507679, 714788, 514128, 9100371, 1116364, 526014, 1031405, 1195942, 1132688, 498402, 510140, 392065, 649755, 547605, 855072, 997762, 1085996, 844376, 390902, 1375283, 1165081, 396401, 999279, 525148, 613596, 705825, 334671, 868520, 740710, 955831, 1697546, 405804, 1663196], "NumGC":31363} Allocated: 1947051128, active: 1948606464, metadata: 22460528 (n_thp 0), resident: 2013569024, mapped: 2028171264, retained: 4009529344 2021-09-13T19:17:58.074-07:00 [Info] memstats {"Alloc":69453320, "TotalAlloc":1622411134832, "Sys":283252984, "Lookups":0, "Mallocs":136226546,"Frees":136208940, "HeapAlloc":69453320, "HeapSys":264011776, "HeapIdle":191676416, "HeapInuse":72335360,"HeapReleased":157614080, "HeapObjects":17606,"GCSys":8912896, "LastGC":1631585877914554295,"PauseTotalNs":62833510030, "PauseNs":[406586, 521602, 1177248, 516528, 464540, 1059970, 710468, 1069779, 622569, 794204, 513988, 904410, 418689, 1022759, 1425604, 1285849, 406992, 1771501, 452976, 449253, 424180, 418234, 1750244, 930777, 1074336, 1328480, 479620, 1336052, 1431843, 1056550, 1020296, 955402, 511455, 1551455, 919829, 416391, 609713, 579311, 1397421, 609700, 370045, 933371, 726471, 848119, 627688, 707872, 390561, 1098826, 517228, 1109345, 457545, 432938, 1185748, 476504, 359177, 521460, 903429, 1118884, 432524, 1143275, 455087, 393056, 1061019, 488510, 1801797, 480195, 836278, 831474, 808998, 1809063, 1563128, 2162005, 1040084, 587060, 482456, 1735358, 808077, 576074, 469630, 469386, 1145596, 1853226, 590495, 392355, 537238, 596466, 1013931, 572327, 906589, 1310205, 421224, 1080171, 1599093, 1044711, 1042418, 843827, 428663, 1037886, 661624, 552405, 1320486, 439790, 507537, 826638, 590283, 1290753, 675932, 1379399, 548799, 895265, 1205475, 771186, 470315, 862845, 439694, 519598, 528547, 882471, 471396, 1388249, 488126, 389972, 586031, 515436, 483445, 418337, 453912, 406958, 366615, 432987, 1014327, 761370, 1090933, 813688, 510022, 1026437, 503894, 495330, 2024184, 713165, 490485, 350040, 928525, 1376749, 604762, 972459, 461425, 512526, 501640, 841498, 1526392, 403403, 1674859, 626582, 368723, 812972, 505473, 856451, 680047, 995210, 410406, 552818, 448354, 542667, 932400, 485556, 951133, 697344, 394990, 543477, 543068, 324969, 947950, 2333659, 594177, 353784, 692382, 1191817, 1268220, 824289, 412306, 529277, 901417, 653991, 1950047, 712111, 917872, 1039740, 424834, 513400, 468379], "NumGC":31554} Allocated: 1942380056, active: 1943937024, metadata: 22460528 (n_thp 0), resident: 2027606016, mapped: 2042208256, retained: 3995492352 Mon Sep 13 19:18:25 PDT 2021 RSS = 1924796 2021-09-13T19:18:28.111-07:00 [Info] memstats {"Alloc":81971624, "TotalAlloc":1630898262008, "Sys":283252984, "Lookups":0, "Mallocs":136956319,"Frees":136936840, "HeapAlloc":81971624, "HeapSys":263946240, "HeapIdle":179175424, "HeapInuse":84770816,"HeapReleased":160768000, "HeapObjects":19479,"GCSys":8978432, "LastGC":1631585908016931805,"PauseTotalNs":62989811006, "PauseNs":[394188, 958298, 694994, 542429, 1081927, 503868, 1043122, 494110, 765932, 1017657, 1011086, 468967, 802481, 890899, 417756, 551035, 1075780, 900749, 1316947, 387922, 2065637, 459323, 1195598, 476474, 844558, 398237, 428643, 451995, 408029, 429379, 975157, 1791914, 506412, 410387, 383671, 623282, 488639, 1117370, 1739348, 510441, 673860, 469894, 1220783, 351045, 1044364, 672229, 1554470, 893388, 892430, 545214, 479889, 586154, 1642663, 563447, 370562, 590233, 323887, 432358, 705234, 1304138, 1204494, 972500, 1573991, 2140931, 419559, 430402, 532672, 1491918, 431812, 959776, 1074763, 392168, 434348, 474574, 536489, 437805, 562889, 1473774, 471388, 760177, 1637353, 1160036, 559921, 1073196, 647660, 1407062, 901403, 506365, 757388, 667479, 476746, 510811, 552188, 393972, 467533, 968739, 1139476, 1311365, 426172, 426372, 1027927, 463640, 543568, 717321, 483065, 443544, 424674, 409027, 1251294, 472673, 1120842, 423697, 447613, 6373171, 381596, 589319, 973548, 662339, 1099592, 536492, 379019, 1576139, 1200194, 365286, 498039, 411262, 603349, 387653, 528767, 807281, 1225465, 819138, 2690638, 1681289, 636794, 1130707, 1255919, 839300, 437222, 474750, 1045459, 1389669, 386884, 389457, 1134283, 452484, 501120, 296949, 616894, 418653, 588458, 783656, 718654, 478403, 982588, 494433, 1426881, 483570, 366964, 900366, 556689, 1134343, 676006, 446153, 333769, 544332, 628196, 432299, 1205946, 2169675, 1075672, 616226, 852667, 507001, 484747, 825065, 565481, 332500, 427157, 453996, 1312316, 364369, 469703, 2338042, 502608, 867008, 520703, 1268368, 1066561, 689684, 571463, 392419, 524393, 1387891], "NumGC":31748} Allocated: 1944766872, active: 1946312704, metadata: 22460528 (n_thp 0), resident: 2031652864, mapped: 2046255104, retained: 3991445504 2021-09-13T19:18:58.139-07:00 [Info] memstats {"Alloc":73172496, "TotalAlloc":1639637713000, "Sys":283252984, "Lookups":0, "Mallocs":137713219,"Frees":137695219, "HeapAlloc":73172496, "HeapSys":263946240, "HeapIdle":187850752, "HeapInuse":76095488,"HeapReleased":152985600, "HeapObjects":18000,"GCSys":8978432, "LastGC":1631585938050574725,"PauseTotalNs":63142565101, "PauseNs":[1181678, 629915, 946114, 930303, 566304, 366730, 851341, 1354312, 1122931, 448365, 396099, 976804, 451636, 791479, 432857, 401014, 482150, 709324, 426891, 947985, 1441488, 491736, 1454431, 660025, 463994, 1734025, 1110471, 372954, 472837, 634194, 1161555, 1084283, 1071517, 738303, 624641, 373668, 1088439, 840997, 1587057, 536254, 482309, 1448541, 524858, 919380, 462703, 755782, 627218, 1114995, 1120056, 987207, 1611503, 467432, 707755, 422948, 383633, 436540, 619676, 354416, 1753242, 436215, 688465, 537814, 540279, 592510, 512092, 1506439, 887101, 534423, 382128, 592450, 378498, 578919, 376084, 412510, 430648, 1011364, 530192, 566118, 1202057, 558841, 1166471, 1580181, 859357, 442371, 841340, 682915, 396090, 718705, 328107, 488344, 359225, 362656, 1105095, 921454, 946578, 863831, 526983, 522149, 480423, 2305509, 501494, 499410, 684414, 594053, 1187777, 726891, 650973, 1797402, 1077975, 1072114, 893701, 1056325, 336183, 435932, 851282, 1203703, 386004, 635193, 1104554, 628314, 1533499, 1260285, 424698, 1026057, 395671, 460106, 622531, 2331988, 841632, 808075, 1266353, 433571, 644758, 958926, 473840, 388742, 1179225, 581941, 368297, 766530, 463933, 1106107, 770113, 342177, 541682, 471445, 636129, 806916, 1236245, 1205161, 389890, 463264, 1084756, 347512, 595838, 1097171, 962646, 576723, 639288, 558191, 1474697, 442661, 957107, 389671, 660082, 1143208, 453974, 347606, 456610, 448741, 445384, 429033, 417043, 845113, 364791, 388779, 397163, 1125755, 900919, 1106755, 409687, 555110, 310316, 506603, 495203, 864159, 535518, 563150, 465982, 458436, 905817, 1051876, 453834, 1162912, 1711379, 1231487, 403176, 498730, 336249, 574196, 580801, 1388550], "NumGC":31950} Allocated: 1953287928, active: 1954832384, metadata: 22460528 (n_thp 0), resident: 2027900928, mapped: 2042503168, retained: 3995197440 Mon Sep 13 19:19:25 PDT 2021 RSS = 1900588 2021-09-13T19:19:28.184-07:00 [Info] memstats {"Alloc":77016704, "TotalAlloc":1648421556560, "Sys":283252984, "Lookups":0, "Mallocs":138469308,"Frees":138451585, "HeapAlloc":77016704, "HeapSys":263946240, "HeapIdle":184066048, "HeapInuse":79880192,"HeapReleased":161890304, "HeapObjects":17723,"GCSys":8978432, "LastGC":1631585968142567042,"PauseTotalNs":63293373296, "PauseNs":[569934, 374797, 413555, 558422, 1013320, 829980, 950664, 520861, 652671, 918751, 687118, 317318, 851275, 1685873, 1493313, 792054, 444783, 428598, 346019, 1117614, 984492, 760189, 414584, 673894, 1257993, 1223010, 1512850, 495250, 417693, 500347, 362930, 556408, 420291, 905406, 457492, 1278908, 1279159, 717415, 869623, 865014, 2042950, 870050, 320833, 445347, 1090392, 389075, 426355, 1435587, 824938, 997049, 346320, 558534, 356330, 1240030, 319814, 419353, 1291678, 437127, 608626, 1384424, 468141, 764709, 418686, 1179312, 730405, 444383, 969711, 863154, 471703, 376968, 484961, 1113355, 680236, 1226342, 972740, 688645, 424143, 609426, 478337, 445574, 560700, 1234002, 636785, 1200870, 566124, 1462441, 1297508, 557038, 450881, 440506, 504407, 501824, 688543, 375512, 327478, 1348940, 470149, 684008, 438553, 929950, 883528, 402348, 903476, 1159813, 813360, 535530, 442152, 1243438, 1124714, 1172538, 580548, 1176917, 922652, 455475, 384982, 414095, 464260, 1160310, 1593249, 615779, 504184, 364955, 385358, 964870, 404482, 470193, 2127749, 856218, 370504, 688195, 565994, 1015718, 485421, 545891, 435190, 443406, 1815074, 1182057, 1139641, 401235, 524953, 1016251, 1181826, 374456, 649009, 494322, 947441, 383805, 360549, 401866, 466201, 1228212, 350756, 483352, 765527, 564014, 1185286, 478159, 485802, 398007, 391458, 539690, 345545, 459653, 547873, 2163269, 1169695, 547972, 396301, 1664206, 491917, 389022, 855209, 1054271, 678516, 1208661, 738088, 684088, 439057, 579945, 1056143, 1505474, 475136, 1441077, 1006868, 732613, 443012, 499967, 352537, 1045474, 375787, 386604, 1395224, 1025249, 840739, 570941, 418181, 431236, 448662, 493146, 455243, 497323], "NumGC":32152} Allocated: 1951616632, active: 1953161216, metadata: 22460528 (n_thp 0), resident: 2025017344, mapped: 2039619584, retained: 3998081024 2021-09-13T19:19:58.224-07:00 [Info] memstats {"Alloc":65490712, "TotalAlloc":1656866559224, "Sys":283252984, "Lookups":0, "Mallocs":139197651,"Frees":139180433, "HeapAlloc":65490712, "HeapSys":264011776, "HeapIdle":195584000, "HeapInuse":68427776,"HeapReleased":159547392, "HeapObjects":17218,"GCSys":8912896, "LastGC":1631585998118292516,"PauseTotalNs":63456378278, "PauseNs":[666721, 681851, 558453, 788774, 420584, 359801, 343852, 1700532, 1246654, 526922, 1247449, 955722, 691251, 427874, 403317, 579250, 586488, 1251115, 964272, 790965, 2899797, 1078856, 1016853, 1696819, 462361, 346171, 1041531, 795367, 1300856, 376845, 439475, 349513, 1469371, 320266, 364760, 1089820, 406266, 1513833, 671688, 853427, 524660, 1181354, 448152, 2441717, 983577, 1187517, 1309179, 650408, 955321, 600098, 642244, 985285, 394724, 830165, 484911, 686519, 852249, 1232013, 744767, 10649884, 871561, 842681, 348677, 537812, 436547, 595805, 842361, 440479, 1042047, 1758254, 451960, 675692, 1371850, 475165, 1600779, 791065, 394198, 638482, 1240449, 442827, 491749, 414267, 489112, 989361, 1028995, 456110, 1229267, 653331, 431387, 1039687, 944137, 433558, 1244436, 415394, 621843, 955195, 418912, 1195613, 709303, 354317, 916106, 394660, 1015685, 928569, 552915, 1275137, 726771, 353385, 354099, 626244, 410867, 383240, 342489, 550762, 455541, 755099, 1080173, 348792, 476979, 423205, 440267, 404350, 1668880, 907805, 423039, 451423, 458751, 838140, 408468, 912827, 350172, 1168233, 408835, 415630, 629670, 455136, 1746678, 1547707, 844596, 1368520, 698653, 396698, 811215, 583390, 489206, 1472805, 376461, 517558, 523200, 360853, 1060787, 974607, 1241970, 413503, 433544, 1008424, 655337, 326591, 1861409, 372102, 401455, 628340, 390077, 406271, 497762, 489276, 721928, 489100, 899244, 1369605, 422044, 626328, 852518, 454607, 534645, 1384487, 413425, 1364052, 866179, 804478, 1007316, 764573, 640962, 719498, 851154, 1162214, 467187, 582304, 2021805, 433080, 576704, 1083446, 679949, 1305891, 443660, 1183153, 482976], "NumGC":32349} Allocated: 1947862944, active: 1949409280, metadata: 22460528 (n_thp 0), resident: 2022281216, mapped: 2036883456, retained: 4000817152 Mon Sep 13 19:20:25 PDT 2021 RSS = 1904252 2021-09-13T19:20:28.252-07:00 [Info] memstats {"Alloc":91094424, "TotalAlloc":1665579351160, "Sys":283252984, "Lookups":0, "Mallocs":139954063,"Frees":139937638, "HeapAlloc":91094424, "HeapSys":263946240, "HeapIdle":169861120, "HeapInuse":94085120,"HeapReleased":158662656, "HeapObjects":16425,"GCSys":8978432, "LastGC":1631586028242394046,"PauseTotalNs":63619459482, "PauseNs":[685721, 615202, 1583191, 390810, 582657, 883880, 435407, 651152, 524948, 433734, 752282, 619346, 488448, 657819, 1179131, 334719, 622255, 1102579, 463281, 1015650, 995623, 452931, 728109, 1448251, 377312, 782104, 1702733, 396001, 373802, 860405, 1244771, 510694, 1311742, 579992, 1017080, 757996, 704897, 1041040, 1392740, 457249, 936285, 1444042, 633253, 1142208, 1144970, 609346, 355562, 579413, 831626, 377325, 475774, 370738, 528742, 359922, 513254, 854232, 374949, 322324, 1143718, 450693, 360584, 561683, 1014471, 935099, 712905, 1688281, 951078, 929463, 4612903, 578301, 424318, 540480, 979366, 951065, 518045, 1969364, 440389, 372423, 1111544, 1046172, 549814, 1402398, 411277, 406049, 389271, 789557, 980324, 1046344, 569718, 684981, 883173, 460896, 823845, 886200, 819100, 340239, 399045, 639553, 432061, 554364, 836515, 662568, 509990, 844367, 325756, 403641, 743717, 862050, 951657, 775373, 462670, 999220, 1434814, 1017224, 983332, 366175, 423857, 412387, 1335487, 374773, 453187, 1847272, 1157151, 422514, 357623, 420029, 1190403, 2018048, 452832, 671777, 344831, 382589, 1070160, 832600, 921177, 1255554, 887837, 491258, 425289, 391323, 294413, 949078, 598010, 599721, 1211789, 387115, 965564, 523106, 4979657, 990158, 633349, 642448, 625382, 565137, 818291, 1233998, 508975, 401946, 709904, 480056, 369553, 740757, 1010645, 447390, 1623745, 650170, 1210080, 1420699, 327238, 1192467, 406458, 424824, 353887, 1072088, 454768, 812271, 996242, 3315618, 397374, 471650, 375959, 512662, 1102380, 1686811, 811037, 408307, 928785, 1559252, 427596, 388454, 397415, 455649, 395307, 866261, 994308, 1134596, 1101464, 1036093, 1219136, 926591, 1045890], "NumGC":32550} Allocated: 1950805144, active: 1952342016, metadata: 22460528 (n_thp 0), resident: 2031931392, mapped: 2046533632, retained: 3991166976 2021-09-13T19:20:58.303-07:00 [Info] memstats {"Alloc":70472912, "TotalAlloc":1673902565232, "Sys":283252984, "Lookups":0, "Mallocs":140673082,"Frees":140655274, "HeapAlloc":70472912, "HeapSys":264011776, "HeapIdle":190627840, "HeapInuse":73383936,"HeapReleased":168943616, "HeapObjects":17808,"GCSys":8912896, "LastGC":1631586058179746009,"PauseTotalNs":63773835890, "PauseNs":[1027808, 1487886, 539894, 423794, 1200504, 845951, 1217037, 663940, 1617929, 517157, 341978, 524125, 1205886, 521531, 848647, 1649449, 808718, 834359, 553372, 397572, 893850, 376988, 492929, 518786, 1048270, 1118266, 797753, 1404663, 933862, 687010, 455750, 365210, 489185, 802323, 872385, 450237, 412204, 1032233, 1449077, 418844, 466885, 1467789, 1188874, 990931, 462814, 495322, 887290, 988764, 1162938, 2664416, 434457, 1054685, 588554, 1154878, 352978, 936983, 393000, 1608724, 350221, 455835, 4339993, 464856, 657286, 1558663, 358887, 1520350, 982661, 1176934, 444566, 613015, 421742, 402817, 885538, 492910, 1122546, 516608, 791712, 392021, 576462, 340100, 461672, 928200, 658383, 395204, 1030313, 462838, 1061115, 361245, 1066417, 605533, 580708, 987982, 435226, 350145, 477419, 570866, 794264, 430471, 476352, 587918, 643399, 582047, 521412, 390253, 699240, 1229034, 882035, 456174, 591546, 417888, 1455560, 711506, 343279, 396900, 451959, 1063201, 710384, 3765026, 390218, 1832978, 1199569, 585251, 712190, 857098, 475723, 1247258, 367087, 921389, 515597, 1036592, 1243265, 389434, 377463, 416427, 502659, 457774, 1647307, 1004738, 668109, 477843, 464588, 604648, 1877878, 457293, 369367, 534644, 962238, 1206741, 548895, 720427, 1234829, 505683, 1037056, 576300, 1223775, 534203, 398322, 623016, 402980, 588771, 477752, 648775, 447146, 617317, 800976, 1059664, 623050, 928064, 372587, 1158179, 411838, 628913, 352722, 358815, 840207, 1328666, 736521, 601735, 736988, 993926, 429671, 385562, 1144885, 388405, 981697, 385111, 784172, 1094278, 1381433, 384217, 1853920, 350246, 807062, 665899, 571905], "NumGC":32745} Allocated: 1942939968, active: 1944477696, metadata: 22460528 (n_thp 0), resident: 2067304448, mapped: 2081906688, retained: 3955793920 Mon Sep 13 19:21:25 PDT 2021 RSS = 1927768 2021-09-13 19:21:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 266, "num_active_sctxs": 93, "num_free_sctxs": 173, "num_trim_sctxs": 19, "num_wctxs": 304, "num_free_wctxs": 241, "num_active_wctxs": 63, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 126, "cpu": 8.43000, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 2018091008, "memory_jemalloc_allocated": 1945845248, "memory_in_use": 2018091008, "memory_stats_size_total": 1639879764, "memory_stats_size_global": 1290613444, "memory_stats_size_page": 1154891180, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 139840, "reclaim_pending": 1376804, "buf_memused": 347833708, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 11, "num_active_smr": 1, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 18081, "total_ops": 522856, "compacts": 2287677, "purges": 223, "lss_data_size": 260030338, "lss_used_space": 3539027682, "lss_fragmentation": 92, "lss_num_reads": 819934, "lss_read_bs": 129287497904, "lss_blk_read_bs": 126190231552, "rlss_num_reads": 30965, "lss_rdr_reads_bs": 10356083306, "lss_blk_rdr_reads_bs": 3829915648, "bytes_written": 115927261184, "bytes_incoming": 34272165088, "write_amp_avg": 1.09, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 15415, "resident_ratio": 85.25524, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 68533572, "recovery_used_space": 969021830, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86443884544, "recovery_lss_frag": 92, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 723.24000 }2021-09-13T19:21:28.344-07:00 [Info] memstats {"Alloc":78897488, "TotalAlloc":1682817778800, "Sys":283252984, "Lookups":0, "Mallocs":141442844,"Frees":141424282, "HeapAlloc":78897488, "HeapSys":264011776, "HeapIdle":182222848, "HeapInuse":81788928,"HeapReleased":152313856, "HeapObjects":18562,"GCSys":8912896, "LastGC":1631586088197497389,"PauseTotalNs":63930642073, "PauseNs":[539820, 1049973, 320451, 1012008, 526978, 441796, 1486223, 403383, 981891, 425051, 454671, 554078, 477617, 656877, 388905, 1142404, 919669, 459333, 1255727, 440334, 478009, 1054114, 477108, 386946, 1515211, 448337, 364997, 922682, 584683, 1274928, 966224, 365524, 762549, 408319, 402625, 538473, 530225, 482857, 456552, 484982, 386372, 920614, 680456, 863859, 411264, 484408, 471869, 1320377, 990414, 1275587, 367493, 400273, 1335737, 386293, 439632, 979592, 673203, 1282838, 571403, 664644, 493528, 913059, 775690, 881465, 555249, 1800854, 2466670, 1614149, 7544221, 1120397, 1018974, 964318, 462469, 667231, 1183755, 440731, 1159862, 1150723, 404241, 1368614, 579882, 1019052, 1355550, 920369, 1193983, 423577, 406055, 1115255, 433597, 433541, 1252722, 1065827, 496063, 1173764, 462228, 824072, 1333544, 1086334, 365630, 1116909, 465983, 370308, 344581, 484798, 304246, 669769, 1167677, 424575, 980004, 464993, 1315329, 1108994, 415194, 775634, 489418, 350134, 417628, 647127, 416494, 749854, 1047077, 2613261, 573761, 306504, 630392, 802558, 734828, 577979, 429881, 1194038, 441509, 783072, 462426, 994494, 559781, 335801, 433343, 1045283, 986165, 511013, 894098, 508225, 565028, 1064617, 491740, 1008427, 1339290, 425627, 394780, 548287, 1285878, 540602, 773858, 345082, 649506, 353664, 516291, 416640, 474259, 867155, 826751, 574364, 1326397, 434827, 1015554, 377245, 404485, 502359, 1035670, 499595, 545731, 517194, 613145, 1327482, 688376, 394916, 1105600, 504669, 495373, 611198, 885511, 1138100, 880960, 357624, 530451, 394768, 421984, 446517, 440514, 430594, 1313160, 573339, 542068, 408626, 444545, 1646274, 550974, 1145309, 879569, 427418, 384625, 356882, 1323464], "NumGC":32948} Allocated: 1947385344, active: 1948921856, metadata: 22460528 (n_thp 0), resident: 2018091008, mapped: 2032693248, retained: 4005007360 2021-09-13 19:21:37 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 19:21:41 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 19:21:41 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 19:21:43 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T19:21:58.384-07:00 [Info] memstats {"Alloc":78069008, "TotalAlloc":1690991486904, "Sys":283252984, "Lookups":0, "Mallocs":142165282,"Frees":142148553, "HeapAlloc":78069008, "HeapSys":263946240, "HeapIdle":182927360, "HeapInuse":81018880,"HeapReleased":141737984, "HeapObjects":16729,"GCSys":8978432, "LastGC":1631586118374249485,"PauseTotalNs":64176634305, "PauseNs":[1186661, 515475, 1343169, 987816, 2350146, 695860, 496742, 1296860, 388421, 742667, 347594, 835669, 1079419, 794245, 1092558, 1950523, 639764, 1134455, 421173, 973208, 1007604, 654529, 1416899, 1579401, 443564, 459889, 604917, 1168604, 911646, 1240086, 498338, 447775, 1119824, 1059014, 1258888, 367789, 556393, 353479, 481138, 500673, 537542, 495365, 450426, 711682, 388311, 1051263, 1631644, 451138, 1412942, 437561, 3050779, 1100272, 1206009, 407071, 771753, 597865, 456663, 1535635, 481748, 550100, 1418303, 476748, 1969659, 3954882, 1970333, 5564445, 4374965, 439686, 5935117, 2176282, 4679859, 697119, 965260, 886644, 1386702, 575424, 369472, 868394, 605316, 1382399, 1322045, 1641147, 804512, 899020, 2396010, 1295851, 463637, 7065357, 975159, 1554932, 491822, 518037, 579611, 519925, 4351629, 1499824, 440273, 1193199, 702285, 1285362, 7811414, 363505, 384958, 727259, 1262130, 347221, 3376247, 2630939, 1774921, 850617, 946053, 353058, 499423, 1181945, 504795, 1153233, 629899, 1531358, 761781, 744352, 1341810, 705757, 1147291, 1086357, 483070, 1048005, 493422, 383876, 502623, 660172, 797292, 1783251, 393149, 489149, 2039112, 397575, 1181727, 955967, 621575, 523979, 421483, 12265602, 1152204, 415891, 764121, 721252, 526677, 533145, 1663077, 610748, 431286, 1383560, 1368323, 508533, 1437208, 509823, 376037, 521019, 1341721, 521400, 4885940, 454660, 395777, 484947, 444733, 694135, 5285968, 349349, 699558, 862279, 1560366, 383955, 425067, 514106, 5134717, 1144502, 894943, 439506, 1170728, 409974, 4457072, 5266381, 4002104, 462913, 1245716, 956962, 456760, 6340447], "NumGC":33136} Allocated: 1709037888, active: 1710567424, metadata: 22460528 (n_thp 0), resident: 1775132672, mapped: 1789734912, retained: 4247965696 Mon Sep 13 19:22:25 PDT 2021 RSS = 1820004 2021-09-13T19:22:28.463-07:00 [Info] memstats {"Alloc":92558744, "TotalAlloc":1699502780936, "Sys":283252984, "Lookups":0, "Mallocs":142904160,"Frees":142885389, "HeapAlloc":92558744, "HeapSys":263913472, "HeapIdle":168443904, "HeapInuse":95469568,"HeapReleased":162111488, "HeapObjects":18771,"GCSys":9011200, "LastGC":1631586148444159181,"PauseTotalNs":64435876500, "PauseNs":[469000, 4658101, 402605, 688134, 1165632, 1125326, 574469, 1582605, 3607324, 550881, 1246513, 717034, 1192460, 422406, 11162702, 1166984, 1171645, 1282048, 1572018, 492808, 2374788, 891401, 5019500, 337821, 939086, 475199, 408963, 6114268, 427385, 5001378, 1179654, 407550, 602357, 714524, 522111, 1786744, 4638616, 483186, 515248, 609524, 571018, 553071, 685804, 994742, 3306690, 3609650, 477720, 1505812, 5385853, 404551, 4836426, 349719, 387129, 476770, 1242378, 4650674, 376971, 802809, 489818, 620694, 2870200, 561755, 734727, 446737, 1286425, 426860, 1717982, 4457654, 899309, 392794, 362159, 797924, 758804, 467608, 430129, 359151, 438375, 563628, 1444015, 631745, 798397, 484666, 1429394, 404648, 1105279, 1671745, 967916, 937762, 991045, 553448, 348434, 589219, 1154129, 993237, 7858968, 424303, 699729, 793058, 414955, 525672, 487633, 740845, 461449, 1126455, 462923, 355816, 443447, 402262, 829161, 406493, 728321, 475159, 591278, 992511, 439919, 609374, 452451, 533397, 551554, 371348, 428080, 4441219, 429282, 841931, 995918, 732983, 982113, 480315, 7257962, 493543, 599977, 441048, 616391, 402119, 3349297, 495309, 2232160, 496171, 732760, 929349, 388745, 610083, 346196, 353175, 627532, 443570, 482228, 684788, 406245, 359183, 870127, 2700133, 955200, 1569748, 1300072, 1257270, 428010, 403748, 413208, 465261, 425759, 9118267, 572736, 472355, 410696, 1076020, 536929, 479596, 921627, 1380773, 505366, 1763220, 493415, 507193, 3768302, 712734, 10248178, 558439, 375493, 546733, 1142867, 805669, 763279, 6080903, 629566, 1563536, 10713223, 336655, 375911, 598848, 442586, 1158497, 2230297], "NumGC":33329} Allocated: 1855467992, active: 1857007616, metadata: 22460528 (n_thp 0), resident: 1920770048, mapped: 1935372288, retained: 4102328320 2021-09-13T19:22:58.505-07:00 [Info] memstats {"Alloc":95986152, "TotalAlloc":1708113383720, "Sys":283252984, "Lookups":0, "Mallocs":143646920,"Frees":143626657, "HeapAlloc":95986152, "HeapSys":264011776, "HeapIdle":165191680, "HeapInuse":98820096,"HeapReleased":157933568, "HeapObjects":20263,"GCSys":8912896, "LastGC":1631586178385935012,"PauseTotalNs":64646926812, "PauseNs":[4625934, 1214739, 542755, 493725, 408855, 481722, 412571, 3767936, 777896, 587523, 1013859, 5357520, 724544, 681634, 1123184, 896688, 400333, 482842, 629564, 391664, 1859965, 1069159, 578105, 1004779, 423794, 5368772, 842817, 502123, 910954, 582220, 377625, 499930, 764235, 1643160, 974158, 1491594, 820222, 1207039, 1644303, 968233, 820490, 6881565, 368506, 334555, 520355, 819773, 439394, 751063, 1190558, 1385620, 1478624, 490280, 630063, 1111663, 531870, 563475, 561434, 312701, 545061, 489866, 448279, 774139, 461422, 494152, 1571728, 885710, 572844, 795620, 434714, 365142, 1038920, 691054, 3111208, 357219, 1710523, 1197771, 367890, 1063171, 637793, 755539, 859312, 460033, 1284357, 1222554, 409420, 3341835, 943437, 524517, 1146905, 1179057, 1766474, 497649, 1136419, 385378, 1001133, 493394, 424721, 385499, 366659, 419039, 688322, 515445, 891235, 397914, 7439908, 522654, 1475810, 433363, 1435465, 374995, 1427325, 985887, 8320889, 1363127, 1226361, 1036411, 460876, 482721, 678022, 656363, 974324, 1207339, 5275596, 512300, 392517, 352979, 354754, 1013001, 1344756, 724168, 362106, 546138, 560268, 1184370, 1015198, 1141259, 762543, 1228490, 481342, 777532, 505816, 2637023, 371307, 1255816, 1055796, 397241, 928146, 621710, 4427258, 389888, 2027693, 1370920, 1062385, 1983124, 384857, 812656, 566020, 1591845, 4519020, 454106, 654929, 536892, 475911, 1187690, 512413, 498552, 996928, 1065667, 448864, 483965, 360950, 455954, 1709223, 445362, 2111976, 565987, 751402, 1577375, 1311621, 595418, 727127, 1033301, 573400, 418445, 488457, 383154, 763360, 607167, 469078, 461792, 1713995, 1189480, 846655, 387142, 1095759], "NumGC":33524} Allocated: 1888755944, active: 1890279424, metadata: 22460528 (n_thp 0), resident: 1977634816, mapped: 1992237056, retained: 4045463552 Mon Sep 13 19:23:25 PDT 2021 RSS = 1894112 2021-09-13T19:23:28.551-07:00 [Info] memstats {"Alloc":68367536, "TotalAlloc":1716472451944, "Sys":283252984, "Lookups":0, "Mallocs":144368754,"Frees":144351531, "HeapAlloc":68367536, "HeapSys":263946240, "HeapIdle":192684032, "HeapInuse":71262208,"HeapReleased":157655040, "HeapObjects":17223,"GCSys":8978432, "LastGC":1631586208478974758,"PauseTotalNs":64822417781, "PauseNs":[919731, 508529, 1845820, 444249, 494179, 1633518, 459197, 648243, 930957, 514052, 848606, 363666, 398485, 491167, 1250632, 470717, 445012, 486577, 562698, 529921, 1441552, 5064504, 347195, 737668, 441081, 379551, 511531, 1102302, 737574, 462318, 2215728, 986894, 965525, 932513, 5259339, 800981, 462047, 1089582, 961903, 981964, 919655, 790097, 391348, 1023152, 456278, 409045, 1001044, 1364807, 425650, 1027195, 382804, 558083, 679231, 3522272, 934153, 1115905, 849874, 940073, 1131286, 452738, 1020899, 545187, 581801, 1109709, 1245709, 426531, 1276488, 334855, 928267, 414461, 537519, 496981, 469626, 1710979, 566513, 608822, 331039, 511131, 389247, 457006, 494164, 378513, 730649, 652729, 513437, 460889, 423059, 704962, 518682, 1554027, 442710, 1084492, 377481, 1131870, 406267, 644660, 409041, 1133175, 455646, 439398, 346756, 570605, 1287418, 383069, 461972, 584390, 397632, 405910, 1322047, 729201, 611346, 388473, 601963, 1114094, 519455, 382796, 1072812, 4571811, 878479, 600150, 1158777, 541363, 1668342, 668401, 11534281, 1007000, 601135, 1039747, 327773, 689971, 536885, 1135687, 1138964, 402316, 390109, 473388, 1164087, 1168236, 596271, 419721, 748258, 2602955, 398575, 1335804, 877058, 495322, 486800, 590437, 501043, 2013516, 1207287, 513868, 373099, 1272979, 574082, 459531, 1136426, 1128340, 868868, 1733654, 524263, 2204590, 629252, 587407, 686829, 1026920, 508473, 1168925, 393465, 379750, 715302, 1161591, 655411, 1976103, 404877, 386831, 610417, 490444, 404249, 372706, 1839017, 512610, 408164, 385085, 998509, 599319, 572045, 894110, 717044, 377293, 624132, 1301046, 664771, 1198350, 479915], "NumGC":33719} Allocated: 1917096400, active: 1918623744, metadata: 22460528 (n_thp 0), resident: 2008256512, mapped: 2022858752, retained: 4014841856 2021-09-13T19:23:58.624-07:00 [Info] memstats {"Alloc":85374608, "TotalAlloc":1725129508872, "Sys":283252984, "Lookups":0, "Mallocs":145117142,"Frees":145099741, "HeapAlloc":85374608, "HeapSys":263913472, "HeapIdle":175644672, "HeapInuse":88268800,"HeapReleased":162119680, "HeapObjects":17401,"GCSys":9011200, "LastGC":1631586238620645000,"PauseTotalNs":64982145432, "PauseNs":[693991, 454743, 742995, 1046807, 346571, 850120, 907441, 352413, 777171, 798085, 1189276, 641156, 360804, 459538, 612873, 605900, 378397, 1299177, 416098, 927159, 552417, 690557, 821965, 462324, 389058, 806118, 2874764, 467673, 1246959, 722261, 467772, 1870706, 854858, 842567, 924118, 669947, 648585, 723137, 1069239, 1212226, 521379, 879673, 1266881, 509929, 346436, 329948, 510194, 642336, 368076, 391474, 463129, 1172165, 362951, 334601, 363061, 339800, 935308, 470123, 390582, 391432, 1309768, 1259314, 1213943, 381896, 478062, 398076, 366657, 492601, 622663, 1905941, 582080, 1001974, 610167, 473620, 1817060, 428354, 1011621, 633707, 907513, 303430, 665609, 484832, 2476760, 1946430, 469557, 1283107, 543053, 397462, 781546, 939704, 1009219, 504857, 631658, 1539150, 405549, 1307525, 1041003, 590429, 385961, 721989, 2106394, 1036892, 981195, 509060, 535256, 434770, 382805, 1057525, 466529, 478932, 1153921, 999256, 1495693, 1059824, 606673, 877020, 950985, 626049, 1096670, 589657, 355878, 355887, 579716, 418239, 400086, 438561, 2497202, 429510, 1319351, 534605, 1859477, 981183, 2511009, 413834, 344856, 581697, 701676, 738016, 645803, 548988, 1264584, 346112, 531158, 1013771, 1276552, 953762, 417574, 505315, 1577835, 1484581, 428634, 434776, 1754956, 814227, 946891, 365044, 499533, 1087016, 724400, 356880, 1044771, 402358, 631353, 775224, 445922, 910693, 959500, 387402, 766345, 393222, 336396, 1044996, 525845, 1724978, 831916, 452187, 471608, 453428, 392852, 516315, 548030, 386887, 1232314, 757464, 561759, 523244, 1144081, 1197675, 893879, 538475, 600264, 1464848, 1197444, 570811, 910464, 834638, 1210575, 2119817, 701939, 490160], "NumGC":33919} Allocated: 1936328352, active: 1937854464, metadata: 22460528 (n_thp 0), resident: 2035056640, mapped: 2049658880, retained: 3988041728 Mon Sep 13 19:24:25 PDT 2021 RSS = 1898668 2021-09-13T19:24:28.655-07:00 [Info] memstats {"Alloc":85355512, "TotalAlloc":1733805322400, "Sys":283252984, "Lookups":0, "Mallocs":145863847,"Frees":145845556, "HeapAlloc":85355512, "HeapSys":263913472, "HeapIdle":175702016, "HeapInuse":88211456,"HeapReleased":154886144, "HeapObjects":18291,"GCSys":9011200, "LastGC":1631586268594022515,"PauseTotalNs":65139015525, "PauseNs":[796687, 618883, 459467, 413162, 829246, 481669, 433325, 355303, 555885, 876240, 613356, 817182, 579493, 589937, 376765, 1023261, 531813, 575421, 704445, 784824, 1031039, 724557, 1187483, 479605, 548317, 454590, 682127, 796255, 1121578, 478884, 393575, 477222, 413178, 1231854, 559612, 1337221, 540324, 764533, 1301146, 456409, 1125468, 403054, 687005, 563790, 889009, 419108, 404361, 1092070, 420323, 743283, 1359367, 433582, 407572, 498691, 369769, 588923, 1952524, 983359, 1833625, 365394, 932256, 492477, 948723, 1615352, 686497, 944720, 307964, 839614, 557657, 456765, 718090, 403388, 833795, 1492424, 396314, 1019879, 425103, 556451, 551529, 660659, 1175387, 496310, 1164905, 345713, 925673, 765633, 1178828, 568258, 492643, 710054, 453547, 466101, 969291, 913660, 406361, 317183, 467772, 1052751, 864216, 968556, 303388, 754789, 426181, 525505, 453138, 398709, 643587, 2350330, 469035, 1415741, 990497, 514680, 475243, 4104450, 825938, 411881, 366811, 654073, 1214713, 508748, 476712, 367465, 382223, 846824, 971895, 371829, 614472, 1342174, 425344, 751123, 382980, 401034, 885650, 368708, 409525, 673454, 467593, 469480, 337435, 317169, 1456431, 1624664, 779711, 672014, 360633, 401761, 522597, 1546275, 1096633, 356872, 516719, 984570, 443164, 1122488, 869737, 589254, 669797, 652948, 459403, 5615478, 407780, 1589390, 337546, 389238, 398052, 432518, 530540, 2052347, 570884, 1021763, 678033, 334014, 1556536, 864185, 449375, 1277941, 397190, 697527, 509099, 442300, 391737, 654894, 868236, 2196725, 444280, 390593, 819103, 856513, 871270, 396341, 990593, 938858, 1422365, 1576752, 1096346, 449226, 830552, 415992, 2103691, 559165, 865051, 505228], "NumGC":34121} Allocated: 1933044168, active: 1934577664, metadata: 22460528 (n_thp 0), resident: 1996685312, mapped: 2011287552, retained: 4026413056 2021-09-13T19:24:58.699-07:00 [Info] memstats {"Alloc":69947624, "TotalAlloc":1742325886848, "Sys":283252984, "Lookups":0, "Mallocs":146595719,"Frees":146577859, "HeapAlloc":69947624, "HeapSys":264011776, "HeapIdle":191209472, "HeapInuse":72802304,"HeapReleased":167370752, "HeapObjects":17860,"GCSys":8912896, "LastGC":1631586298591335572,"PauseTotalNs":65293882610, "PauseNs":[409065, 1194560, 715987, 465689, 696758, 836532, 1254614, 1626328, 805142, 1238669, 383396, 398656, 749316, 1205489, 1613503, 1168728, 543023, 486457, 1016941, 592521, 461392, 1121468, 412368, 806786, 360236, 364215, 1420112, 751916, 630466, 704131, 369298, 589376, 1058726, 1123724, 419546, 505881, 1144218, 608039, 494004, 1295660, 840314, 1483308, 4098404, 1900001, 384382, 355509, 525341, 464837, 843339, 1484820, 471199, 367948, 436838, 380257, 526536, 496890, 905800, 371891, 1697699, 1057583, 799288, 549671, 350185, 1297015, 659856, 474467, 475679, 722715, 1044166, 990760, 629739, 872152, 405716, 413127, 1011870, 1166874, 579575, 588911, 1354017, 1204031, 1175684, 1717929, 1657103, 410243, 516605, 944378, 1022221, 1101113, 376017, 456013, 1328699, 1219963, 981128, 1418487, 422688, 719411, 864268, 436341, 332842, 491125, 468692, 388683, 1359073, 431630, 567967, 682582, 326193, 518519, 388338, 453906, 509602, 473174, 969267, 1252850, 654735, 425597, 408822, 983991, 793944, 444943, 359163, 1247207, 828063, 339458, 373342, 731882, 377253, 350778, 344953, 452461, 1394205, 306947, 549343, 533395, 445154, 1280204, 377091, 722836, 858857, 407567, 616723, 449864, 852335, 425140, 376718, 1476027, 1208474, 682995, 485467, 1585032, 443682, 744087, 516522, 967020, 2249790, 439482, 478975, 1105587, 446740, 677576, 967727, 1233686, 826323, 1110909, 1432930, 463191, 1417240, 1196784, 1145078, 528149, 1025601, 425990, 1849450, 477580, 370989, 802695, 462050, 403505, 362915, 879037, 495942, 850008, 909031, 489708, 487916, 516299, 417278, 955715, 578457, 874797, 1324789, 754002, 372387, 376644, 1164921, 805674, 388602, 394279], "NumGC":34319} Allocated: 1946118208, active: 1947652096, metadata: 22460528 (n_thp 0), resident: 2059010048, mapped: 2073612288, retained: 3964088320 Mon Sep 13 19:25:25 PDT 2021 RSS = 1914200 2021-09-13T19:25:28.744-07:00 [Info] memstats {"Alloc":82406928, "TotalAlloc":1751047552624, "Sys":283252984, "Lookups":0, "Mallocs":147348754,"Frees":147331703, "HeapAlloc":82406928, "HeapSys":263913472, "HeapIdle":178585600, "HeapInuse":85327872,"HeapReleased":161325056, "HeapObjects":17051,"GCSys":9011200, "LastGC":1631586328729541503,"PauseTotalNs":65449839114, "PauseNs":[348012, 394785, 412876, 578227, 657679, 930252, 392718, 721724, 349206, 347992, 943185, 476006, 1161743, 422584, 364749, 934985, 933510, 424651, 818423, 349799, 485984, 508727, 429184, 1131993, 518626, 975285, 541563, 1209351, 339752, 434071, 567030, 989533, 808895, 1108005, 882579, 1525725, 1185327, 409000, 422183, 574275, 539591, 673524, 1097636, 879789, 607034, 1062250, 432007, 786041, 535399, 505305, 877860, 1780681, 1091395, 454331, 526815, 510959, 1163713, 423354, 428274, 1239844, 1745289, 603816, 746604, 1068935, 474771, 862058, 892056, 1073534, 372552, 446992, 366888, 354766, 886171, 1023603, 368816, 445160, 1249602, 510960, 764506, 494219, 1145198, 573129, 1499103, 394214, 564803, 448520, 1127806, 1435892, 721231, 468777, 994127, 431674, 882148, 410690, 1105388, 625572, 431891, 533298, 582612, 882165, 1120351, 847283, 1103740, 685698, 551015, 321273, 335205, 699798, 1017432, 625715, 389282, 1299785, 1018701, 556733, 415064, 404032, 543400, 991283, 384658, 394754, 1003268, 415532, 556918, 449019, 592036, 544807, 2002539, 320661, 1084777, 1162450, 476730, 777271, 1216861, 384877, 1442762, 447109, 965021, 1366012, 669867, 590377, 463846, 455811, 812157, 523954, 375473, 599070, 829872, 381602, 1103096, 462663, 410770, 821692, 1133959, 376451, 3420856, 991271, 893386, 413691, 879553, 527948, 450029, 1092421, 410111, 509482, 603737, 406401, 534185, 823374, 905789, 1048406, 563990, 702197, 442359, 406867, 1610777, 1060612, 845127, 449742, 1143176, 444381, 472001, 1787993, 6721336, 842703, 453294, 410426, 409275, 387162, 530326, 1270114, 1056452, 545646, 460587, 1709230, 1015607, 464066, 1285876, 682724, 510334, 579733, 749094, 472613], "NumGC":34521} Allocated: 1956143800, active: 1957679104, metadata: 22460528 (n_thp 0), resident: 2019655680, mapped: 2034257920, retained: 4003442688 2021-09-13T19:25:58.774-07:00 [Info] memstats {"Alloc":60993288, "TotalAlloc":1759526161576, "Sys":283252984, "Lookups":0, "Mallocs":148079630,"Frees":148063436, "HeapAlloc":60993288, "HeapSys":263913472, "HeapIdle":200007680, "HeapInuse":63905792,"HeapReleased":153894912, "HeapObjects":16194,"GCSys":9011200, "LastGC":1631586358755523893,"PauseTotalNs":65597512791, "PauseNs":[490936, 363968, 505676, 455411, 402594, 400011, 897733, 527334, 388211, 452766, 574760, 603623, 393368, 635770, 1087420, 1094085, 887248, 534749, 1646273, 353213, 345932, 611237, 1297612, 699358, 810442, 538492, 422290, 1530953, 469803, 1645977, 418193, 568572, 373142, 401886, 672112, 471889, 474384, 347259, 398733, 788655, 1204317, 1143354, 1448337, 3740730, 309821, 528988, 1002153, 1556175, 1868141, 441919, 388455, 397938, 1222290, 954017, 938803, 456039, 441321, 538330, 376806, 927280, 914136, 712903, 1068895, 1608712, 1201044, 468797, 328049, 386161, 945767, 1003928, 464701, 1138025, 599009, 400651, 376176, 392714, 702297, 420759, 1108271, 1053425, 517248, 1355900, 1002032, 448296, 645523, 1267118, 1011467, 849472, 418061, 322660, 933870, 991583, 1938723, 1057113, 588280, 372813, 538579, 449574, 1105522, 446879, 373310, 535788, 590056, 828921, 384688, 478819, 463858, 1120208, 567813, 844060, 847891, 390809, 421085, 672012, 1862522, 395271, 414284, 947347, 342259, 487482, 1771448, 788383, 558020, 1057416, 308867, 1213706, 395635, 482634, 493916, 463326, 441220, 449177, 394830, 1706089, 559202, 379714, 785540, 457372, 684387, 439253, 886287, 415813, 1109898, 558753, 809106, 528327, 930213, 1046484, 627837, 493260, 629222, 971808, 881334, 1743842, 1076936, 361453, 496349, 582485, 371165, 314465, 1039117, 412571, 575707, 914147, 903338, 1134703, 491538, 352735, 326568, 478292, 418226, 472727, 381147, 850262, 996354, 450965, 388839, 359336, 602835, 2206366, 777701, 468074, 525102, 1847657, 421761, 935310, 484456, 439375, 947157, 420032, 452459, 440290, 598137, 419800, 1807824, 1116634, 971556, 527021, 1261486], "NumGC":34720} Allocated: 1946149112, active: 1947684864, metadata: 22460528 (n_thp 0), resident: 2053832704, mapped: 2068434944, retained: 3969265664 Mon Sep 13 19:26:25 PDT 2021 RSS = 1900112 2021-09-13 19:26:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 266, "num_active_sctxs": 92, "num_free_sctxs": 174, "num_trim_sctxs": 19, "num_wctxs": 306, "num_free_wctxs": 244, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.38400, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 2024996864, "memory_jemalloc_allocated": 1948942944, "memory_in_use": 2024996864, "memory_stats_size_total": 1643706340, "memory_stats_size_global": 1293297976, "memory_stats_size_page": 1157578976, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 139840, "reclaim_pending": 1311216, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 18122, "total_ops": 549361, "compacts": 2423360, "purges": 223, "lss_data_size": 260030338, "lss_used_space": 4001900258, "lss_fragmentation": 93, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116390133760, "bytes_incoming": 36009514828, "write_amp_avg": 1.04, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 15456, "resident_ratio": 85.28860, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 68533572, "recovery_used_space": 1078847878, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86553710592, "recovery_lss_frag": 93, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 724.88000 }2021-09-13T19:26:28.810-07:00 [Info] memstats {"Alloc":65943144, "TotalAlloc":1768206119184, "Sys":283252984, "Lookups":0, "Mallocs":148825074,"Frees":148807646, "HeapAlloc":65943144, "HeapSys":263946240, "HeapIdle":195149824, "HeapInuse":68796416,"HeapReleased":150487040, "HeapObjects":17428,"GCSys":8978432, "LastGC":1631586388723153572,"PauseTotalNs":65743910398, "PauseNs":[413718, 587802, 735538, 1007524, 523458, 461711, 372011, 776465, 430694, 1181368, 447893, 332899, 904382, 929029, 1688052, 392728, 865548, 342348, 814409, 1123371, 927299, 416632, 375502, 532142, 488370, 877592, 1131982, 408722, 1138145, 491792, 397015, 996631, 1027107, 352292, 377249, 1234996, 955004, 792372, 649767, 460052, 401482, 473863, 536088, 1336987, 1100843, 808418, 367351, 1128231, 1481595, 1276406, 342128, 488530, 907494, 930259, 865988, 423430, 398367, 548698, 2079034, 1159224, 395946, 1025580, 729766, 412719, 644930, 1234859, 935668, 339912, 490243, 1408234, 1169711, 1573875, 486415, 1108606, 487608, 432637, 561254, 872477, 472283, 923195, 1126292, 337243, 1035346, 709215, 401457, 405084, 529948, 448434, 507369, 987132, 589853, 419653, 386225, 819482, 516848, 648964, 1173372, 457661, 1134069, 948095, 936583, 591832, 530604, 1091514, 947059, 1180753, 796882, 716631, 742207, 437547, 501153, 871847, 393590, 1249286, 2750040, 427779, 404516, 581102, 453086, 673861, 731721, 374620, 824095, 622147, 496627, 442471, 722799, 431438, 425562, 975403, 376871, 931899, 1006359, 885605, 571553, 511849, 524227, 370719, 505714, 357305, 521882, 1725165, 400117, 354019, 1329862, 767975, 797675, 683119, 400020, 433201, 406511, 1212700, 406808, 1057509, 1457860, 636199, 455683, 598347, 409215, 427137, 388137, 690030, 957748, 561075, 594427, 433463, 378854, 994818, 1086495, 943228, 413724, 546832, 905396, 1019258, 356471, 359092, 682338, 439667, 962726, 1202051, 562363, 1360416, 591943, 902914, 629499, 1010763, 1894106, 722370, 1091031, 403117, 458198, 832229, 272287, 463978, 416427, 442685, 990139, 1130134, 755631], "NumGC":34919} Allocated: 1947370080, active: 1948913664, metadata: 22460528 (n_thp 0), resident: 2031681536, mapped: 2046283776, retained: 3991416832 2021-09-13T19:26:58.840-07:00 [Info] memstats {"Alloc":75514016, "TotalAlloc":1777081746280, "Sys":283252984, "Lookups":0, "Mallocs":149590652,"Frees":149573082, "HeapAlloc":75514016, "HeapSys":263913472, "HeapIdle":185458688, "HeapInuse":78454784,"HeapReleased":164216832, "HeapObjects":17570,"GCSys":9011200, "LastGC":1631586418811436655,"PauseTotalNs":65898081871, "PauseNs":[441443, 644520, 355789, 614373, 1125840, 650212, 777404, 345818, 517329, 746514, 555646, 1371899, 375215, 1335704, 466068, 402134, 409471, 1085869, 478963, 402871, 1045280, 1250113, 891359, 490427, 620785, 901402, 538272, 977763, 871800, 377414, 747390, 518930, 1015715, 423707, 884266, 394244, 330730, 945406, 349665, 1603394, 808912, 1178748, 394085, 370629, 1281566, 695664, 408644, 614840, 630509, 387666, 517944, 914205, 508362, 566071, 744847, 1004457, 658697, 482440, 607139, 1174113, 712851, 885888, 1114197, 1021992, 404414, 514741, 461220, 386632, 530051, 443834, 597847, 2995648, 1046703, 480543, 534047, 467826, 707098, 904262, 394191, 512922, 418310, 528391, 384495, 405904, 312414, 549513, 1315943, 328632, 1218836, 781522, 390627, 1002824, 862328, 605696, 446322, 561454, 710862, 573764, 782902, 347132, 768738, 848211, 364696, 1181468, 434141, 1342717, 1235206, 462850, 507665, 722444, 1501043, 796641, 429640, 846753, 656830, 460737, 677308, 987436, 1181771, 1746962, 406667, 522428, 396793, 468492, 481892, 407109, 392436, 952936, 1835230, 374970, 505577, 788424, 477790, 681492, 2047586, 798731, 429424, 893821, 467047, 898947, 439807, 740588, 946452, 953909, 974711, 1365050, 648580, 1758444, 2731558, 1984510, 467453, 525257, 555445, 358997, 1062707, 1001722, 605766, 531118, 481488, 525907, 590290, 394027, 399003, 1260892, 391167, 589099, 2192987, 1151771, 563083, 506836, 524779, 908169, 473457, 1552805, 583018, 396777, 458592, 399159, 650636, 1165647, 1524797, 587757, 508637, 860499, 361519, 398791, 426165, 384371, 1183679, 381552, 406974, 551616, 1193407, 458191, 546429, 398127, 1687534, 1441754, 421914, 1008695, 1558883, 444767, 704655, 735140, 927448], "NumGC":35124} Allocated: 1946451880, active: 1947996160, metadata: 22460528 (n_thp 0), resident: 2009137152, mapped: 2023739392, retained: 4013961216 Mon Sep 13 19:27:25 PDT 2021 RSS = 1936788 2021-09-13T19:27:28.871-07:00 [Info] memstats {"Alloc":64298320, "TotalAlloc":1785588244496, "Sys":283252984, "Lookups":0, "Mallocs":150323322,"Frees":150305728, "HeapAlloc":64298320, "HeapSys":264011776, "HeapIdle":196861952, "HeapInuse":67149824,"HeapReleased":139165696, "HeapObjects":17594,"GCSys":8912896, "LastGC":1631586448737816090,"PauseTotalNs":66037473097, "PauseNs":[530293, 918652, 992235, 405356, 965491, 375729, 461268, 613572, 407508, 1212891, 443989, 1383363, 318842, 436233, 470586, 413109, 477815, 445699, 542342, 1222996, 527439, 869783, 395394, 944619, 586085, 541058, 557404, 1362941, 715548, 874624, 426265, 501768, 1470514, 373219, 382685, 1550956, 1342130, 466786, 402818, 424195, 393422, 346543, 426720, 410131, 1519256, 583987, 918790, 408937, 944988, 623398, 710420, 594310, 478829, 441269, 1134497, 440006, 706370, 737905, 447047, 353261, 515551, 441541, 1052887, 810476, 500056, 677948, 510403, 1656742, 594811, 378476, 776709, 395501, 620268, 592370, 417281, 699884, 409361, 849738, 1101557, 1012712, 598277, 710627, 463686, 593023, 680226, 417059, 1147637, 569857, 360517, 900806, 495010, 900606, 618202, 462460, 1196901, 431520, 1153680, 1101039, 1506845, 507208, 730589, 566833, 460495, 1232644, 705100, 990654, 1215194, 427006, 1543709, 985520, 998412, 442243, 943809, 748797, 437647, 826480, 1005427, 524165, 396972, 551769, 1225751, 415297, 928743, 532883, 1047432, 1563131, 444153, 1143040, 588405, 481687, 581673, 505635, 916353, 562157, 971539, 1120032, 358618, 643634, 401683, 520956, 358172, 426351, 383230, 560096, 425437, 617783, 513383, 1212660, 468293, 546521, 441417, 791396, 1747964, 1090555, 512948, 418940, 886281, 379256, 502622, 1253966, 913044, 962764, 1072037, 641842, 1027829, 512495, 422441, 829677, 461714, 712339, 1030446, 334243, 2034668, 440317, 439675, 1080106, 483178, 462518, 639713, 1239269, 380059, 721736, 2319992, 399697, 1480249, 354126, 476392, 644448, 421134, 496299, 485610, 462280, 903106, 678501], "NumGC":35318} Allocated: 1948675136, active: 1950203904, metadata: 22460528 (n_thp 0), resident: 2052681728, mapped: 2067283968, retained: 3970416640 2021-09-13T19:27:58.917-07:00 [Info] memstats {"Alloc":74839592, "TotalAlloc":1793784736368, "Sys":283252984, "Lookups":0, "Mallocs":151031201,"Frees":151012759, "HeapAlloc":74839592, "HeapSys":264011776, "HeapIdle":186327040, "HeapInuse":77684736,"HeapReleased":158629888, "HeapObjects":18442,"GCSys":8912896, "LastGC":1631586478794502492,"PauseTotalNs":66180827638, "PauseNs":[484472, 517759, 774002, 738655, 448078, 399170, 644706, 475250, 756721, 1250424, 563232, 880339, 623356, 724901, 470274, 722494, 411057, 1119841, 893517, 477340, 951715, 903950, 525786, 419096, 534927, 2274415, 1285463, 343230, 1365954, 366180, 1842519, 617021, 1010431, 1174364, 406696, 389163, 1630812, 699753, 1177004, 945960, 1236214, 1372547, 626790, 563140, 875683, 537353, 501940, 368228, 411075, 599016, 1275431, 1300732, 1191550, 418136, 903703, 1860889, 428757, 489457, 745284, 1796918, 1145899, 1333403, 286991, 353155, 440357, 948062, 1469851, 459115, 299980, 361939, 396575, 755064, 438812, 397152, 426606, 2204623, 495657, 1420027, 332086, 1015447, 1653714, 494530, 414108, 759871, 419174, 557908, 394868, 804306, 484055, 1227515, 502377, 569077, 801882, 543225, 395604, 1036274, 1180587, 1485564, 936439, 548252, 433448, 700939, 555130, 763640, 1225026, 537320, 556297, 1360411, 608700, 756327, 354098, 849828, 753991, 506008, 639263, 420370, 395056, 614427, 389221, 918685, 888626, 1103794, 725393, 581234, 419954, 1395965, 505719, 493175, 604503, 848024, 377507, 331921, 999074, 985062, 1923109, 1715192, 1216946, 574906, 832208, 525314, 407861, 849248, 472862, 865834, 553192, 741002, 477928, 424083, 634352, 375468, 484090, 377328, 456248, 474225, 830456, 541575, 447280, 470524, 440178, 1204066, 933195, 1119068, 567908, 421991, 504497, 1106073, 375381, 346426, 1119971, 326389, 394427, 426495, 469170, 364416, 1040726, 578684, 550077, 907597, 758416, 1359408, 660199, 940394, 758048, 509481, 763428, 367638, 1011346, 390845, 1039507, 584053, 423867, 388808], "NumGC":35510} Allocated: 1954474112, active: 1956003840, metadata: 22460528 (n_thp 0), resident: 2031284224, mapped: 2045886464, retained: 3991814144 Mon Sep 13 19:28:26 PDT 2021 RSS = 1895472 2021-09-13T19:28:28.966-07:00 [Info] memstats {"Alloc":69914088, "TotalAlloc":1802250352904, "Sys":283252984, "Lookups":0, "Mallocs":151759234,"Frees":151741613, "HeapAlloc":69914088, "HeapSys":264011776, "HeapIdle":191135744, "HeapInuse":72876032,"HeapReleased":163545088, "HeapObjects":17621,"GCSys":8912896, "LastGC":1631586508853972670,"PauseTotalNs":66341410668, "PauseNs":[1211747, 387109, 711558, 649849, 1238050, 388932, 926889, 373838, 745323, 518622, 586688, 1016262, 443207, 1028334, 390985, 1012623, 1025548, 594852, 459601, 538020, 992611, 410398, 623827, 435832, 528124, 677335, 464371, 1320558, 407976, 630755, 917650, 1685043, 511745, 1674490, 507216, 848258, 393611, 588287, 422406, 406205, 619883, 387480, 486069, 397688, 2142961, 480211, 994424, 378450, 545215, 459479, 1284892, 2187842, 398027, 404842, 559514, 1133906, 1059415, 432465, 855407, 749528, 1129655, 1237203, 406703, 361003, 487051, 545445, 2019744, 633479, 636089, 1341902, 351841, 1335936, 442306, 394775, 1507692, 752380, 484851, 959017, 349535, 1219699, 538309, 622269, 734597, 411505, 729284, 1011177, 798780, 676598, 332379, 1257742, 927510, 367542, 11470687, 625052, 1348060, 1216705, 480159, 517174, 890767, 408180, 515878, 902636, 402081, 1127003, 790880, 660481, 958716, 753686, 391421, 1070252, 356861, 553237, 1046888, 521978, 285057, 480676, 527232, 432099, 338329, 1022712, 1368202, 706150, 745813, 400166, 312624, 406817, 389864, 617550, 827319, 402349, 800345, 1595195, 901337, 1163277, 1175593, 768803, 516848, 746428, 1168783, 512332, 4799871, 949815, 859384, 1130284, 334306, 326337, 825359, 806337, 928805, 766923, 460468, 404151, 368125, 2119249, 412189, 1643629, 1038689, 495468, 937782, 415375, 1276336, 933881, 538588, 884241, 1823065, 492355, 974888, 420131, 467301, 408985, 1384040, 1302554, 874695, 393049, 598518, 409491, 533461, 885540, 466919, 561162, 497159, 1587377, 550795, 482738, 394324, 400073, 469900, 448691, 919281, 482533, 767525, 883490, 516683, 406736, 394279, 452587], "NumGC":35706} Allocated: 1945296440, active: 1946828800, metadata: 22460528 (n_thp 0), resident: 2017521664, mapped: 2032123904, retained: 4005576704 2021-09-13T19:28:59.009-07:00 [Info] memstats {"Alloc":88145328, "TotalAlloc":1810955841480, "Sys":283252984, "Lookups":0, "Mallocs":152510116,"Frees":152492148, "HeapAlloc":88145328, "HeapSys":263946240, "HeapIdle":172916736, "HeapInuse":91029504,"HeapReleased":167084032, "HeapObjects":17968,"GCSys":8978432, "LastGC":1631586538996568050,"PauseTotalNs":66497733169, "PauseNs":[1207271, 724342, 1117964, 720520, 561697, 1160151, 1618854, 483314, 876038, 1348053, 367003, 383013, 1592364, 336830, 1473747, 666589, 319617, 356725, 1102196, 374775, 442167, 1790912, 499799, 1192426, 844405, 823375, 462308, 478255, 1040007, 354887, 639187, 505322, 483717, 1158534, 1247692, 616813, 469324, 326722, 1193118, 1016067, 1555323, 516255, 601876, 1378493, 1834240, 525901, 747576, 1148004, 561784, 1043721, 477389, 373827, 1174194, 462940, 441067, 1194760, 354325, 935637, 1152464, 475563, 397651, 861960, 547315, 873429, 1223772, 513362, 361222, 424433, 347612, 795986, 416474, 678340, 481685, 992862, 466702, 840142, 397165, 631147, 490122, 443232, 495079, 464367, 363047, 427051, 435549, 373552, 406448, 625045, 437165, 453746, 390759, 475032, 1719133, 337899, 427563, 571207, 579912, 631077, 365321, 2814241, 552677, 502500, 1312310, 1675245, 624270, 353275, 452113, 420322, 1024647, 417172, 495625, 452707, 1015695, 1318284, 795514, 436102, 420451, 585276, 863658, 5972190, 1533096, 557191, 385398, 358987, 1018068, 383895, 914846, 613140, 446877, 975019, 1022697, 452870, 471616, 397397, 497424, 473012, 1148341, 352923, 380472, 545479, 415408, 494617, 625106, 448201, 693110, 620160, 374942, 1075029, 652769, 789721, 1224250, 416600, 479614, 480137, 1181903, 852507, 975149, 915947, 419205, 1185134, 358329, 555175, 628188, 1206935, 423423, 482540, 586353, 915598, 472060, 936482, 1020986, 1290505, 1149721, 434765, 326316, 716917, 1036766, 600178, 936929, 472499, 513572, 1059618, 766995, 837506, 931264, 556729, 770327, 1565622, 392810, 1063810, 1419674, 1047248, 771951, 1192360, 479597, 493095, 1034520, 1081560, 432855, 1336953, 1011455, 353334, 1411446], "NumGC":35909} Allocated: 1955567496, active: 1957101568, metadata: 22460528 (n_thp 0), resident: 2012803072, mapped: 2027405312, retained: 4010295296 Mon Sep 13 19:29:26 PDT 2021 RSS = 1909976 2021-09-13T19:29:29.047-07:00 [Info] memstats {"Alloc":72183768, "TotalAlloc":1819549821208, "Sys":283252984, "Lookups":0, "Mallocs":153249600,"Frees":153231572, "HeapAlloc":72183768, "HeapSys":264011776, "HeapIdle":188940288, "HeapInuse":75071488,"HeapReleased":155557888, "HeapObjects":18028,"GCSys":8912896, "LastGC":1631586568942312868,"PauseTotalNs":66656870795, "PauseNs":[429384, 316811, 479510, 431185, 821490, 657446, 525169, 395244, 661220, 1692575, 781479, 1511706, 1488032, 960874, 357513, 939469, 779552, 413925, 1083808, 1016891, 1264186, 938210, 597899, 437349, 406209, 334308, 403208, 449115, 289261, 369196, 2305496, 566064, 1072806, 328054, 1091612, 846315, 424184, 2017085, 670857, 1034419, 501416, 471545, 1165144, 487856, 1218439, 454977, 499084, 765742, 470023, 439925, 1145769, 694617, 1249964, 1243098, 485459, 427800, 359742, 1307092, 1110059, 580024, 511052, 535084, 534069, 1413712, 792100, 1174069, 426626, 1595334, 823086, 388174, 618964, 958357, 1300388, 951985, 1192404, 1649723, 666022, 1022626, 327139, 402944, 404791, 662922, 862740, 2843523, 1396802, 1177304, 632375, 406944, 656503, 1258560, 912480, 444185, 876293, 1049075, 483459, 499971, 449858, 1243120, 532580, 1086971, 440716, 411687, 942375, 1126388, 482846, 1223599, 464293, 1453300, 1082713, 458130, 928609, 446178, 1594575, 1191740, 527020, 1191380, 390915, 486273, 373359, 596155, 454581, 1317021, 426437, 1263022, 396445, 362246, 561518, 987706, 650868, 1050940, 1216698, 633908, 1394006, 1364555, 406982, 431810, 870931, 928245, 416824, 537873, 490742, 1188418, 568521, 604718, 625170, 910145, 441318, 564695, 1149226, 458333, 1159891, 457817, 995827, 391225, 961405, 819452, 763015, 543008, 369535, 523915, 635741, 1066007, 398377, 1789438, 1394056, 444499, 1568330, 641932, 1064588, 677118, 775700, 434869, 1770393, 539661, 1033184, 1108349, 879431, 423757, 939856, 430752, 375250, 1384739, 1362747, 834981, 797285, 646427, 475525, 474311, 531323, 1616009, 648388, 985231, 1079053, 442502, 980296, 1185073], "NumGC":36105} Allocated: 1955085392, active: 1956610048, metadata: 22460528 (n_thp 0), resident: 2047930368, mapped: 2062532608, retained: 3975168000 2021-09-13T19:29:59.079-07:00 [Info] memstats {"Alloc":83417824, "TotalAlloc":1828321895912, "Sys":283252984, "Lookups":0, "Mallocs":154003562,"Frees":153984400, "HeapAlloc":83417824, "HeapSys":264011776, "HeapIdle":177815552, "HeapInuse":86196224,"HeapReleased":157794304, "HeapObjects":19162,"GCSys":8912896, "LastGC":1631586598938267368,"PauseTotalNs":66799775161, "PauseNs":[1216772, 566791, 515191, 764896, 741201, 1299100, 1258778, 1595179, 715492, 1077456, 492639, 381545, 1236061, 350484, 447376, 516097, 442517, 676522, 538079, 1660539, 1023220, 578060, 1187590, 614773, 1339714, 1075324, 841485, 968640, 624760, 822396, 465725, 363948, 560559, 483896, 739551, 446560, 355319, 865406, 404425, 637721, 356975, 432613, 1395765, 560797, 708834, 984824, 1258156, 424898, 1164824, 339599, 477087, 518490, 379126, 756057, 616706, 442790, 387232, 446607, 765173, 2403573, 445676, 642150, 672692, 867656, 497409, 533185, 962315, 497690, 364901, 585132, 401903, 936989, 886669, 463916, 452315, 427112, 1051911, 848947, 387881, 466608, 431076, 1160318, 734240, 444179, 476692, 444425, 361987, 415512, 866614, 455739, 481595, 832683, 395476, 478556, 387102, 1182760, 724570, 831153, 472820, 583532, 385857, 506879, 628973, 1604260, 456116, 1036868, 1196989, 435531, 526279, 484988, 647605, 387127, 943108, 381659, 439861, 370658, 353585, 447710, 899316, 327450, 496859, 1114989, 858813, 1347311, 372251, 1126757, 1035982, 519855, 408800, 421914, 507013, 1638876, 1212871, 963117, 395922, 434358, 525338, 448619, 851280, 331119, 396025, 463027, 698075, 918469, 1735731, 950623, 622179, 371058, 860048, 397326, 468858, 429503, 388229, 518275, 396397, 732194, 715881, 965223, 432833, 1020212, 423826, 973101, 435168, 1067084, 352134, 531300, 918207, 1274306, 393284, 797934, 400954, 416413, 1334871, 1113287, 550387, 408138, 767649, 696141, 490989, 434571, 1276986, 503773, 640070, 1189227, 1498123, 923565, 418818, 752390, 1357290, 1204235, 394595, 1100383, 486367, 381985, 382155, 337948, 563973, 683230, 1257214, 450935, 512968, 627108, 962256], "NumGC":36308} Allocated: 1954072432, active: 1955586048, metadata: 22460528 (n_thp 0), resident: 2058817536, mapped: 2073419776, retained: 3964280832 Mon Sep 13 19:30:26 PDT 2021 RSS = 1910940 2021-09-13T19:30:29.125-07:00 [Info] memstats {"Alloc":61184976, "TotalAlloc":1837044974104, "Sys":283252984, "Lookups":0, "Mallocs":154759573,"Frees":154742367, "HeapAlloc":61184976, "HeapSys":264011776, "HeapIdle":199925760, "HeapInuse":64086016,"HeapReleased":165478400, "HeapObjects":17206,"GCSys":8912896, "LastGC":1631586629018674285,"PauseTotalNs":66962465610, "PauseNs":[1942353, 415115, 450313, 571737, 598331, 1655038, 579118, 1575908, 425633, 954156, 645411, 1052620, 323518, 438947, 402848, 1319443, 491712, 544122, 908229, 521529, 689705, 408317, 740509, 1440681, 336778, 394986, 1168253, 910262, 1478620, 613596, 324471, 502948, 1025400, 691606, 950874, 846610, 2045504, 747730, 946650, 1305600, 818142, 442560, 1078019, 1291457, 809999, 1134992, 724810, 409073, 903724, 362292, 370094, 685410, 355048, 1367260, 653105, 673810, 410117, 765839, 1490900, 382616, 2217434, 584911, 509619, 463953, 498438, 392779, 897994, 433097, 519542, 942843, 461486, 1561320, 727855, 1464386, 416690, 961819, 909675, 327009, 971584, 393007, 779568, 518778, 961299, 850478, 611447, 627119, 1242259, 1372942, 425120, 430653, 580487, 581673, 410203, 1095032, 880920, 538359, 1091968, 398665, 1157707, 785705, 820981, 734948, 846091, 1803710, 434770, 579018, 1170458, 1025693, 429688, 933785, 638422, 460736, 851975, 767592, 1680766, 481120, 607121, 438840, 1623689, 1628011, 1239914, 1104005, 709227, 1242640, 342934, 338659, 389914, 558428, 588974, 573108, 418276, 425652, 332251, 1143613, 453375, 448211, 1580092, 1662116, 374886, 508084, 464457, 520425, 738807, 482609, 699248, 671331, 408151, 586941, 1188141, 813771, 825122, 451593, 1318902, 968951, 1207600, 389779, 1016002, 1411754, 1207737, 869385, 1198899, 564514, 392731, 467928, 941709, 447793, 1016720, 438827, 1208114, 369936, 402099, 443025, 509391, 977163, 1007278, 472715, 346628, 376392, 546016, 539158, 395245, 368274, 1128548, 1387571, 876515, 1335002, 414681, 396956, 529126, 7692274, 866321, 417464, 556310, 482716, 414546, 931927, 528316, 463963, 874317, 1071726, 341996, 406869], "NumGC":36510} Allocated: 1963475544, active: 1964978176, metadata: 22460528 (n_thp 0), resident: 2036572160, mapped: 2051174400, retained: 3986526208 2021-09-13T19:30:59.162-07:00 [Info] memstats {"Alloc":82782208, "TotalAlloc":1845710471584, "Sys":283252984, "Lookups":0, "Mallocs":155507491,"Frees":155489558, "HeapAlloc":82782208, "HeapSys":263946240, "HeapIdle":178274304, "HeapInuse":85671936,"HeapReleased":157302784, "HeapObjects":17933,"GCSys":8978432, "LastGC":1631586659117040712,"PauseTotalNs":67112660931, "PauseNs":[1172317, 416751, 333990, 892234, 1023102, 428975, 1071476, 603180, 847227, 496429, 439253, 448268, 1233995, 406119, 337276, 469396, 427911, 347974, 1413016, 943433, 697153, 467365, 812722, 347439, 819151, 2879149, 778313, 1049897, 886383, 678098, 933808, 1620290, 476774, 933405, 810389, 368967, 493412, 378724, 622955, 1878732, 719233, 349964, 700179, 653040, 726084, 944653, 567053, 363377, 611025, 505835, 377077, 1918396, 1536204, 634541, 413835, 550978, 370951, 443763, 404194, 368652, 1504157, 441946, 378160, 632853, 513737, 365036, 398879, 446670, 934465, 436618, 429249, 509478, 803467, 1791956, 452147, 1560994, 1790040, 585933, 401530, 869257, 825585, 1330500, 466483, 521208, 1108284, 406066, 777213, 789400, 548321, 390434, 454048, 928848, 1026498, 474062, 526293, 471804, 613330, 394065, 1300909, 1125185, 890304, 1496196, 400054, 408308, 424285, 405050, 670367, 623636, 1166181, 750847, 654496, 720807, 520382, 1516632, 411924, 1175245, 557523, 753284, 1557674, 336069, 533074, 740036, 1695542, 641461, 364487, 355007, 442275, 768836, 390869, 502406, 882201, 919943, 865065, 401865, 439347, 425487, 754741, 1359300, 958004, 380042, 1742208, 490130, 1015817, 946902, 699065, 551811, 698614, 485582, 441486, 1494007, 354514, 996516, 599206, 477652, 383581, 1433591, 438816, 955169, 1736535, 1009615, 361239, 383093, 766028, 981983, 729063, 827232, 377726, 394764, 522923, 570741, 370943, 388090, 1168717, 388495, 431004, 386591, 870961, 481505, 598904, 586994, 998878, 1250956, 531349, 954234, 405325, 1295165, 1111929, 778887, 475846, 707577, 1236817, 390098, 374628, 1828997, 335794, 470433, 447502, 1270220, 1560545, 1022311, 381175], "NumGC":36711} Allocated: 1954987032, active: 1956491264, metadata: 22460528 (n_thp 0), resident: 2016124928, mapped: 2031644672, retained: 4006055936 Mon Sep 13 19:31:26 PDT 2021 RSS = 1920008 2021-09-13 19:31:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 266, "num_active_sctxs": 92, "num_free_sctxs": 174, "num_trim_sctxs": 19, "num_wctxs": 306, "num_free_wctxs": 244, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.45200, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 2082234368, "memory_jemalloc_allocated": 1950759072, "memory_in_use": 2082234368, "memory_stats_size_total": 1645410844, "memory_stats_size_global": 1294346872, "memory_stats_size_page": 1158627872, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 139840, "reclaim_pending": 1966824, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 18138, "total_ops": 576187, "compacts": 2560410, "purges": 223, "lss_data_size": 260030338, "lss_used_space": 4001900258, "lss_fragmentation": 93, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116390133760, "bytes_incoming": 37767905476, "write_amp_avg": 0.98, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 15472, "resident_ratio": 85.30158, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 68533572, "recovery_used_space": 1078847878, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86553710592, "recovery_lss_frag": 93, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 725.52000 }2021-09-13T19:31:29.193-07:00 [Info] memstats {"Alloc":72295664, "TotalAlloc":1854164987936, "Sys":283252984, "Lookups":0, "Mallocs":156234922,"Frees":156218523, "HeapAlloc":72295664, "HeapSys":263913472, "HeapIdle":188702720, "HeapInuse":75210752,"HeapReleased":149823488, "HeapObjects":16399,"GCSys":9011200, "LastGC":1631586689177921541,"PauseTotalNs":67259069084, "PauseNs":[445815, 1068227, 434497, 1477191, 697070, 431356, 338587, 370608, 689918, 839277, 481994, 648102, 466242, 499692, 396316, 584486, 781066, 455307, 412491, 879029, 1086729, 1598578, 1212920, 447524, 471190, 509099, 389485, 345315, 375129, 911515, 1922477, 522746, 913369, 519979, 1328771, 396584, 457020, 707958, 1360055, 541981, 359925, 474533, 415187, 434404, 509519, 556070, 689919, 857373, 430124, 570540, 1019440, 379597, 343133, 538574, 419954, 934772, 515748, 1032727, 951312, 897844, 403441, 1038949, 884010, 1302372, 468611, 1020940, 1006480, 399918, 496503, 904838, 368696, 680095, 654893, 595460, 454339, 1231345, 465856, 1248979, 624450, 597803, 419616, 476285, 564952, 463253, 1386193, 982568, 545707, 909486, 463426, 707414, 421963, 1078353, 1760537, 1020170, 1184374, 581571, 447462, 472194, 1405344, 890262, 429208, 473404, 913569, 1037815, 439162, 681704, 441788, 1062152, 1600361, 325600, 552099, 953843, 449297, 370246, 858861, 873068, 437696, 1263071, 945901, 369851, 808211, 813855, 459376, 526734, 1186033, 375337, 691678, 595792, 504908, 1150772, 1188718, 728238, 762309, 1078051, 718374, 543767, 1245088, 2158184, 786731, 464454, 723159, 897066, 451178, 1045530, 1295410, 468556, 448590, 397119, 627386, 471185, 470578, 1793587, 525302, 352675, 809324, 1182592, 844928, 395560, 977781, 500069, 426210, 489031, 426205, 564664, 639937, 1288668, 492467, 555361, 431500, 655217, 1038870, 1471930, 1163033, 815968, 391165, 412101, 1190263, 1778313, 405028, 621903, 560798, 433691, 505951, 869114, 713292, 940022, 381839, 393293, 1576830, 924420, 450170, 474520, 401342, 499923, 1752030, 1089175, 1025217, 440313], "NumGC":36909} Allocated: 1946565344, active: 1948069888, metadata: 22460528 (n_thp 0), resident: 2082234368, mapped: 2096836608, retained: 3940864000 2021-09-13 19:31:33 [gsi-test] [7] Rollback() - PREPARE... 2021-09-13 19:31:33 [gsi-test] [7] Rollback() - DO!!! 2021-09-13 19:31:39 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 19:31:43 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 19:31:43 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 19:31:45 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T19:31:59.226-07:00 [Info] memstats {"Alloc":72025312, "TotalAlloc":1854723129152, "Sys":283252984, "Lookups":0, "Mallocs":156317842,"Frees":156279558, "HeapAlloc":72025312, "HeapSys":264011776, "HeapIdle":189333504, "HeapInuse":74678272,"HeapReleased":176726016, "HeapObjects":38284,"GCSys":8912896, "LastGC":1631586700140697922,"PauseTotalNs":67269378783, "PauseNs":[725431, 437816, 1898947, 898154, 345001, 1203735, 449498, 446664, 427605, 1716489, 370655, 465519, 924185], "NumGC":36922} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:32:26 PDT 2021 RSS = 1861632 2021-09-13T19:32:29.254-07:00 [Info] memstats {"Alloc":73557848, "TotalAlloc":1854724661688, "Sys":283252984, "Lookups":0, "Mallocs":156336668,"Frees":156279573, "HeapAlloc":73557848, "HeapSys":264011776, "HeapIdle":187826176, "HeapInuse":76185600,"HeapReleased":176726016, "HeapObjects":57095,"GCSys":8912896, "LastGC":1631586700140697922,"PauseTotalNs":67269378783, "PauseNs":[], "NumGC":36922} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:32:59.285-07:00 [Info] memstats {"Alloc":75073544, "TotalAlloc":1854726177384, "Sys":283252984, "Lookups":0, "Mallocs":156355445,"Frees":156279582, "HeapAlloc":75073544, "HeapSys":264011776, "HeapIdle":186195968, "HeapInuse":77815808,"HeapReleased":176726016, "HeapObjects":75863,"GCSys":8912896, "LastGC":1631586700140697922,"PauseTotalNs":67269378783, "PauseNs":[], "NumGC":36922} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:33:26 PDT 2021 RSS = 1861632 2021-09-13T19:33:29.314-07:00 [Info] memstats {"Alloc":39024264, "TotalAlloc":1854727691104, "Sys":283252984, "Lookups":0, "Mallocs":156374260,"Frees":156353141, "HeapAlloc":39024264, "HeapSys":264011776, "HeapIdle":218955776, "HeapInuse":45056000,"HeapReleased":178249728, "HeapObjects":21119,"GCSys":8912896, "LastGC":1631586808083413429,"PauseTotalNs":67269734060, "PauseNs":[355277], "NumGC":36923} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:33:59.344-07:00 [Info] memstats {"Alloc":40553360, "TotalAlloc":1854729220200, "Sys":283252984, "Lookups":0, "Mallocs":156393048,"Frees":156353150, "HeapAlloc":40553360, "HeapSys":264011776, "HeapIdle":218398720, "HeapInuse":45613056,"HeapReleased":178249728, "HeapObjects":39898,"GCSys":8912896, "LastGC":1631586808083413429,"PauseTotalNs":67269734060, "PauseNs":[], "NumGC":36923} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:34:26 PDT 2021 RSS = 1861632 2021-09-13T19:34:29.369-07:00 [Info] memstats {"Alloc":42092600, "TotalAlloc":1854730759440, "Sys":283252984, "Lookups":0, "Mallocs":156411859,"Frees":156353159, "HeapAlloc":42092600, "HeapSys":264011776, "HeapIdle":217866240, "HeapInuse":46145536,"HeapReleased":178249728, "HeapObjects":58700,"GCSys":8912896, "LastGC":1631586808083413429,"PauseTotalNs":67269734060, "PauseNs":[], "NumGC":36923} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:34:59.398-07:00 [Info] memstats {"Alloc":43639960, "TotalAlloc":1854732306800, "Sys":283252984, "Lookups":0, "Mallocs":156430636,"Frees":156353168, "HeapAlloc":43639960, "HeapSys":264011776, "HeapIdle":217251840, "HeapInuse":46759936,"HeapReleased":178249728, "HeapObjects":77468,"GCSys":8912896, "LastGC":1631586808083413429,"PauseTotalNs":67269734060, "PauseNs":[], "NumGC":36923} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:35:26 PDT 2021 RSS = 1861632 2021-09-13T19:35:29.433-07:00 [Info] memstats {"Alloc":39499384, "TotalAlloc":1854733834376, "Sys":283252984, "Lookups":0, "Mallocs":156449457,"Frees":156422386, "HeapAlloc":39499384, "HeapSys":264011776, "HeapIdle":218185728, "HeapInuse":45826048,"HeapReleased":178249728, "HeapObjects":27071,"GCSys":8912896, "LastGC":1631586928095441966,"PauseTotalNs":67270096248, "PauseNs":[362188], "NumGC":36924} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:35:59.469-07:00 [Info] memstats {"Alloc":41025024, "TotalAlloc":1854735360016, "Sys":283252984, "Lookups":0, "Mallocs":156468237,"Frees":156422395, "HeapAlloc":41025024, "HeapSys":264011776, "HeapIdle":217653248, "HeapInuse":46358528,"HeapReleased":178249728, "HeapObjects":45842,"GCSys":8912896, "LastGC":1631586928095441966,"PauseTotalNs":67270096248, "PauseNs":[], "NumGC":36924} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:36:26 PDT 2021 RSS = 1861632 2021-09-13 19:36:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.17200, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4497807074, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886040576, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1197894022, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672756736, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }2021-09-13T19:36:29.496-07:00 [Info] memstats {"Alloc":42608360, "TotalAlloc":1854736943352, "Sys":283252984, "Lookups":0, "Mallocs":156487277,"Frees":156422442, "HeapAlloc":42608360, "HeapSys":264011776, "HeapIdle":216981504, "HeapInuse":47030272,"HeapReleased":178249728, "HeapObjects":64835,"GCSys":8912896, "LastGC":1631586928095441966,"PauseTotalNs":67270096248, "PauseNs":[], "NumGC":36924} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:36:59.527-07:00 [Info] memstats {"Alloc":44105032, "TotalAlloc":1854738440024, "Sys":283252984, "Lookups":0, "Mallocs":156506059,"Frees":156422451, "HeapAlloc":44105032, "HeapSys":264011776, "HeapIdle":216399872, "HeapInuse":47611904,"HeapReleased":178249728, "HeapObjects":83608,"GCSys":8912896, "LastGC":1631586928095441966,"PauseTotalNs":67270096248, "PauseNs":[], "NumGC":36924} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:37:26 PDT 2021 RSS = 1861632 2021-09-13T19:37:29.554-07:00 [Info] memstats {"Alloc":39978920, "TotalAlloc":1854739934848, "Sys":283252984, "Lookups":0, "Mallocs":156524862,"Frees":156491837, "HeapAlloc":39978920, "HeapSys":264011776, "HeapIdle":217653248, "HeapInuse":46358528,"HeapReleased":178249728, "HeapObjects":33025,"GCSys":8912896, "LastGC":1631587048110118833,"PauseTotalNs":67270482639, "PauseNs":[386391], "NumGC":36925} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:37:59.581-07:00 [Info] memstats {"Alloc":41467840, "TotalAlloc":1854741423768, "Sys":283252984, "Lookups":0, "Mallocs":156543642,"Frees":156491846, "HeapAlloc":41467840, "HeapSys":264011776, "HeapIdle":217104384, "HeapInuse":46907392,"HeapReleased":178249728, "HeapObjects":51796,"GCSys":8912896, "LastGC":1631587048110118833,"PauseTotalNs":67270482639, "PauseNs":[], "NumGC":36925} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:38:26 PDT 2021 RSS = 1861632 2021-09-13T19:38:29.609-07:00 [Info] memstats {"Alloc":43002024, "TotalAlloc":1854742957952, "Sys":283252984, "Lookups":0, "Mallocs":156562460,"Frees":156491855, "HeapAlloc":43002024, "HeapSys":264011776, "HeapIdle":216555520, "HeapInuse":47456256,"HeapReleased":178249728, "HeapObjects":70605,"GCSys":8912896, "LastGC":1631587048110118833,"PauseTotalNs":67270482639, "PauseNs":[], "NumGC":36925} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:38:59.642-07:00 [Info] memstats {"Alloc":44550984, "TotalAlloc":1854744506912, "Sys":283252984, "Lookups":0, "Mallocs":156581238,"Frees":156491864, "HeapAlloc":44550984, "HeapSys":264011776, "HeapIdle":215908352, "HeapInuse":48103424,"HeapReleased":178249728, "HeapObjects":89374,"GCSys":8912896, "LastGC":1631587048110118833,"PauseTotalNs":67270482639, "PauseNs":[], "NumGC":36925} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:39:26 PDT 2021 RSS = 1861632 2021-09-13T19:39:29.669-07:00 [Info] memstats {"Alloc":40450072, "TotalAlloc":1854746056040, "Sys":283252984, "Lookups":0, "Mallocs":156600058,"Frees":156561076, "HeapAlloc":40450072, "HeapSys":264011776, "HeapIdle":217243648, "HeapInuse":46768128,"HeapReleased":178249728, "HeapObjects":38982,"GCSys":8912896, "LastGC":1631587168127567448,"PauseTotalNs":67270790462, "PauseNs":[307823], "NumGC":36926} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:39:59.700-07:00 [Info] memstats {"Alloc":42030960, "TotalAlloc":1854747636928, "Sys":283252984, "Lookups":0, "Mallocs":156618844,"Frees":156561085, "HeapAlloc":42030960, "HeapSys":264011776, "HeapIdle":216588288, "HeapInuse":47423488,"HeapReleased":178249728, "HeapObjects":57759,"GCSys":8912896, "LastGC":1631587168127567448,"PauseTotalNs":67270790462, "PauseNs":[], "NumGC":36926} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:40:26 PDT 2021 RSS = 1861632 2021-09-13T19:40:29.729-07:00 [Info] memstats {"Alloc":45616784, "TotalAlloc":1854751222752, "Sys":283252984, "Lookups":0, "Mallocs":156641097,"Frees":156561697, "HeapAlloc":45616784, "HeapSys":264011776, "HeapIdle":213827584, "HeapInuse":50184192,"HeapReleased":178249728, "HeapObjects":79400,"GCSys":8912896, "LastGC":1631587168127567448,"PauseTotalNs":67270790462, "PauseNs":[], "NumGC":36926} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:40:59.760-07:00 [Info] memstats {"Alloc":47200448, "TotalAlloc":1854752806416, "Sys":283252984, "Lookups":0, "Mallocs":156659886,"Frees":156561706, "HeapAlloc":47200448, "HeapSys":264011776, "HeapIdle":213172224, "HeapInuse":50839552,"HeapReleased":178249728, "HeapObjects":98180,"GCSys":8912896, "LastGC":1631587168127567448,"PauseTotalNs":67270790462, "PauseNs":[], "NumGC":36926} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:41:26 PDT 2021 RSS = 1861632 2021-09-13 19:41:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.39600, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4497807074, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886040576, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1197894022, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672756736, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }2021-09-13T19:41:29.796-07:00 [Info] memstats {"Alloc":40957680, "TotalAlloc":1854754448152, "Sys":283252984, "Lookups":0, "Mallocs":156678930,"Frees":156633945, "HeapAlloc":40957680, "HeapSys":264011776, "HeapIdle":216645632, "HeapInuse":47366144,"HeapReleased":178249728, "HeapObjects":44985,"GCSys":8912896, "LastGC":1631587288146555520,"PauseTotalNs":67271156803, "PauseNs":[366341], "NumGC":36927} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 19:41:40 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 19:41:44 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 19:41:44 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 19:41:46 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T19:41:59.835-07:00 [Info] memstats {"Alloc":44427496, "TotalAlloc":1854757917968, "Sys":283252984, "Lookups":0, "Mallocs":156699165,"Frees":156634001, "HeapAlloc":44427496, "HeapSys":264011776, "HeapIdle":214237184, "HeapInuse":49774592,"HeapReleased":178249728, "HeapObjects":65164,"GCSys":8912896, "LastGC":1631587288146555520,"PauseTotalNs":67271156803, "PauseNs":[], "NumGC":36927} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:42:26 PDT 2021 RSS = 1861632 2021-09-13T19:42:29.873-07:00 [Info] memstats {"Alloc":45961584, "TotalAlloc":1854759452056, "Sys":283252984, "Lookups":0, "Mallocs":156717998,"Frees":156634010, "HeapAlloc":45961584, "HeapSys":264011776, "HeapIdle":213663744, "HeapInuse":50348032,"HeapReleased":178249728, "HeapObjects":83988,"GCSys":8912896, "LastGC":1631587288146555520,"PauseTotalNs":67271156803, "PauseNs":[], "NumGC":36927} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:42:59.900-07:00 [Info] memstats {"Alloc":47508000, "TotalAlloc":1854760998472, "Sys":283252984, "Lookups":0, "Mallocs":156736779,"Frees":156634019, "HeapAlloc":47508000, "HeapSys":264011776, "HeapIdle":213057536, "HeapInuse":50954240,"HeapReleased":178249728, "HeapObjects":102760,"GCSys":8912896, "LastGC":1631587288146555520,"PauseTotalNs":67271156803, "PauseNs":[], "NumGC":36927} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:43:27 PDT 2021 RSS = 1861632 2021-09-13T19:43:29.932-07:00 [Info] memstats {"Alloc":41405480, "TotalAlloc":1854762555904, "Sys":283252984, "Lookups":0, "Mallocs":156755607,"Frees":156704664, "HeapAlloc":41405480, "HeapSys":264011776, "HeapIdle":216317952, "HeapInuse":47693824,"HeapReleased":178249728, "HeapObjects":50943,"GCSys":8912896, "LastGC":1631587408172575525,"PauseTotalNs":67271490962, "PauseNs":[334159], "NumGC":36928} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:43:59.960-07:00 [Info] memstats {"Alloc":42947888, "TotalAlloc":1854764098312, "Sys":283252984, "Lookups":0, "Mallocs":156774390,"Frees":156704673, "HeapAlloc":42947888, "HeapSys":264011776, "HeapIdle":215687168, "HeapInuse":48324608,"HeapReleased":178249728, "HeapObjects":69717,"GCSys":8912896, "LastGC":1631587408172575525,"PauseTotalNs":67271490962, "PauseNs":[], "NumGC":36928} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:44:27 PDT 2021 RSS = 1861632 2021-09-13T19:44:29.988-07:00 [Info] memstats {"Alloc":44495688, "TotalAlloc":1854765646112, "Sys":283252984, "Lookups":0, "Mallocs":156793214,"Frees":156704682, "HeapAlloc":44495688, "HeapSys":264011776, "HeapIdle":215130112, "HeapInuse":48881664,"HeapReleased":178249728, "HeapObjects":88532,"GCSys":8912896, "LastGC":1631587408172575525,"PauseTotalNs":67271490962, "PauseNs":[], "NumGC":36928} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:45:00.013-07:00 [Info] memstats {"Alloc":46028600, "TotalAlloc":1854767179024, "Sys":283252984, "Lookups":0, "Mallocs":156811989,"Frees":156704691, "HeapAlloc":46028600, "HeapSys":264011776, "HeapIdle":214573056, "HeapInuse":49438720,"HeapReleased":178249728, "HeapObjects":107298,"GCSys":8912896, "LastGC":1631587408172575525,"PauseTotalNs":67271490962, "PauseNs":[], "NumGC":36928} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:45:27 PDT 2021 RSS = 1861632 2021-09-13T19:45:30.040-07:00 [Info] memstats {"Alloc":41880520, "TotalAlloc":1854768728152, "Sys":283252984, "Lookups":0, "Mallocs":156830816,"Frees":156773931, "HeapAlloc":41880520, "HeapSys":264011776, "HeapIdle":215834624, "HeapInuse":48177152,"HeapReleased":178249728, "HeapObjects":56885,"GCSys":8912896, "LastGC":1631587528200183005,"PauseTotalNs":67271843011, "PauseNs":[352049], "NumGC":36929} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:46:00.067-07:00 [Info] memstats {"Alloc":43421056, "TotalAlloc":1854770268688, "Sys":283252984, "Lookups":0, "Mallocs":156849605,"Frees":156773940, "HeapAlloc":43421056, "HeapSys":264011776, "HeapIdle":215195648, "HeapInuse":48816128,"HeapReleased":178249728, "HeapObjects":75665,"GCSys":8912896, "LastGC":1631587528200183005,"PauseTotalNs":67271843011, "PauseNs":[], "NumGC":36929} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:46:27 PDT 2021 RSS = 1861632 2021-09-13 19:46:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.34000, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4497872610, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886106112, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1197926790, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672789504, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }2021-09-13T19:46:30.095-07:00 [Info] memstats {"Alloc":45040120, "TotalAlloc":1854771887752, "Sys":283252984, "Lookups":0, "Mallocs":156868656,"Frees":156773987, "HeapAlloc":45040120, "HeapSys":264011776, "HeapIdle":214507520, "HeapInuse":49504256,"HeapReleased":178249728, "HeapObjects":94669,"GCSys":8912896, "LastGC":1631587528200183005,"PauseTotalNs":67271843011, "PauseNs":[], "NumGC":36929} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:47:00.122-07:00 [Info] memstats {"Alloc":46588168, "TotalAlloc":1854773435800, "Sys":283252984, "Lookups":0, "Mallocs":156887436,"Frees":156773996, "HeapAlloc":46588168, "HeapSys":264011776, "HeapIdle":213958656, "HeapInuse":50053120,"HeapReleased":178249728, "HeapObjects":113440,"GCSys":8912896, "LastGC":1631587528200183005,"PauseTotalNs":67271843011, "PauseNs":[], "NumGC":36929} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:47:27 PDT 2021 RSS = 1861632 2021-09-13T19:47:30.148-07:00 [Info] memstats {"Alloc":42365832, "TotalAlloc":1854775019456, "Sys":283252984, "Lookups":0, "Mallocs":156906255,"Frees":156843411, "HeapAlloc":42365832, "HeapSys":264011776, "HeapIdle":215302144, "HeapInuse":48709632,"HeapReleased":178249728, "HeapObjects":62844,"GCSys":8912896, "LastGC":1631587648226229532,"PauseTotalNs":67272341289, "PauseNs":[498278], "NumGC":36930} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:48:00.184-07:00 [Info] memstats {"Alloc":43949264, "TotalAlloc":1854776602888, "Sys":283252984, "Lookups":0, "Mallocs":156925049,"Frees":156843420, "HeapAlloc":43949264, "HeapSys":264011776, "HeapIdle":214687744, "HeapInuse":49324032,"HeapReleased":178249728, "HeapObjects":81629,"GCSys":8912896, "LastGC":1631587648226229532,"PauseTotalNs":67272341289, "PauseNs":[], "NumGC":36930} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:48:27 PDT 2021 RSS = 1861632 2021-09-13T19:48:30.212-07:00 [Info] memstats {"Alloc":45512296, "TotalAlloc":1854778165920, "Sys":283252984, "Lookups":0, "Mallocs":156943873,"Frees":156843429, "HeapAlloc":45512296, "HeapSys":264011776, "HeapIdle":214106112, "HeapInuse":49905664,"HeapReleased":178249728, "HeapObjects":100444,"GCSys":8912896, "LastGC":1631587648226229532,"PauseTotalNs":67272341289, "PauseNs":[], "NumGC":36930} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:49:00.240-07:00 [Info] memstats {"Alloc":47074536, "TotalAlloc":1854779728160, "Sys":283252984, "Lookups":0, "Mallocs":156962656,"Frees":156843438, "HeapAlloc":47074536, "HeapSys":264011776, "HeapIdle":213516288, "HeapInuse":50495488,"HeapReleased":178249728, "HeapObjects":119218,"GCSys":8912896, "LastGC":1631587648226229532,"PauseTotalNs":67272341289, "PauseNs":[], "NumGC":36930} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:49:27 PDT 2021 RSS = 1861632 2021-09-13T19:49:30.273-07:00 [Info] memstats {"Alloc":42829848, "TotalAlloc":1854781284424, "Sys":283252984, "Lookups":0, "Mallocs":156981482,"Frees":156912694, "HeapAlloc":42829848, "HeapSys":264011776, "HeapIdle":214851584, "HeapInuse":49160192,"HeapReleased":178249728, "HeapObjects":68788,"GCSys":8912896, "LastGC":1631587768251974886,"PauseTotalNs":67272693865, "PauseNs":[352576], "NumGC":36931} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:50:00.299-07:00 [Info] memstats {"Alloc":44389296, "TotalAlloc":1854782843872, "Sys":283252984, "Lookups":0, "Mallocs":157000244,"Frees":156912703, "HeapAlloc":44389296, "HeapSys":264011776, "HeapIdle":214351872, "HeapInuse":49659904,"HeapReleased":178249728, "HeapObjects":87541,"GCSys":8912896, "LastGC":1631587768251974886,"PauseTotalNs":67272693865, "PauseNs":[], "NumGC":36931} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:50:27 PDT 2021 RSS = 1861632 2021-09-13T19:50:30.325-07:00 [Info] memstats {"Alloc":47967480, "TotalAlloc":1854786422056, "Sys":283252984, "Lookups":0, "Mallocs":157022517,"Frees":156913317, "HeapAlloc":47967480, "HeapSys":264011776, "HeapIdle":211542016, "HeapInuse":52469760,"HeapReleased":178249728, "HeapObjects":109200,"GCSys":8912896, "LastGC":1631587768251974886,"PauseTotalNs":67272693865, "PauseNs":[], "NumGC":36931} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:51:00.352-07:00 [Info] memstats {"Alloc":49522984, "TotalAlloc":1854787977560, "Sys":283252984, "Lookups":0, "Mallocs":157041294,"Frees":156913326, "HeapAlloc":49522984, "HeapSys":264011776, "HeapIdle":210903040, "HeapInuse":53108736,"HeapReleased":178249728, "HeapObjects":127968,"GCSys":8912896, "LastGC":1631587768251974886,"PauseTotalNs":67272693865, "PauseNs":[], "NumGC":36931} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:51:27 PDT 2021 RSS = 1861632 2021-09-13 19:51:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.33800, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4497872610, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886106112, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1197926790, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672789504, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }2021-09-13T19:51:30.382-07:00 [Info] memstats {"Alloc":43332208, "TotalAlloc":1854789570976, "Sys":283252984, "Lookups":0, "Mallocs":157060341,"Frees":156985579, "HeapAlloc":43332208, "HeapSys":264011776, "HeapIdle":214302720, "HeapInuse":49709056,"HeapReleased":178249728, "HeapObjects":74762,"GCSys":8912896, "LastGC":1631587888294287156,"PauseTotalNs":67273055527, "PauseNs":[361662], "NumGC":36932} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 19:51:40 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 19:51:44 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 19:51:44 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 19:51:47 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T19:52:00.410-07:00 [Info] memstats {"Alloc":46806888, "TotalAlloc":1854793045656, "Sys":283252984, "Lookups":0, "Mallocs":157080562,"Frees":156985635, "HeapAlloc":46806888, "HeapSys":264011776, "HeapIdle":211820544, "HeapInuse":52191232,"HeapReleased":178249728, "HeapObjects":94927,"GCSys":8912896, "LastGC":1631587888294287156,"PauseTotalNs":67273055527, "PauseNs":[], "NumGC":36932} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:52:27 PDT 2021 RSS = 1861632 2021-09-13T19:52:30.436-07:00 [Info] memstats {"Alloc":48349616, "TotalAlloc":1854794588384, "Sys":283252984, "Lookups":0, "Mallocs":157099388,"Frees":156985644, "HeapAlloc":48349616, "HeapSys":264011776, "HeapIdle":211279872, "HeapInuse":52731904,"HeapReleased":178249728, "HeapObjects":113744,"GCSys":8912896, "LastGC":1631587888294287156,"PauseTotalNs":67273055527, "PauseNs":[], "NumGC":36932} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:53:00.465-07:00 [Info] memstats {"Alloc":49892784, "TotalAlloc":1854796131552, "Sys":283252984, "Lookups":0, "Mallocs":157118165,"Frees":156985653, "HeapAlloc":49892784, "HeapSys":264011776, "HeapIdle":210649088, "HeapInuse":53362688,"HeapReleased":178249728, "HeapObjects":132512,"GCSys":8912896, "LastGC":1631587888294287156,"PauseTotalNs":67273055527, "PauseNs":[], "NumGC":36932} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:53:27 PDT 2021 RSS = 1861632 2021-09-13T19:53:30.494-07:00 [Info] memstats {"Alloc":43787112, "TotalAlloc":1854797663960, "Sys":283252984, "Lookups":0, "Mallocs":157136990,"Frees":157056287, "HeapAlloc":43787112, "HeapSys":264011776, "HeapIdle":213860352, "HeapInuse":50151424,"HeapReleased":178249728, "HeapObjects":80703,"GCSys":8912896, "LastGC":1631588008328737479,"PauseTotalNs":67273393686, "PauseNs":[338159], "NumGC":36933} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:54:00.521-07:00 [Info] memstats {"Alloc":45321680, "TotalAlloc":1854799198528, "Sys":283252984, "Lookups":0, "Mallocs":157155774,"Frees":157056296, "HeapAlloc":45321680, "HeapSys":264011776, "HeapIdle":213286912, "HeapInuse":50724864,"HeapReleased":178249728, "HeapObjects":99478,"GCSys":8912896, "LastGC":1631588008328737479,"PauseTotalNs":67273393686, "PauseNs":[], "NumGC":36933} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:54:27 PDT 2021 RSS = 1861632 2021-09-13T19:54:30.548-07:00 [Info] memstats {"Alloc":46876440, "TotalAlloc":1854800753288, "Sys":283252984, "Lookups":0, "Mallocs":157174595,"Frees":157056305, "HeapAlloc":46876440, "HeapSys":264011776, "HeapIdle":212680704, "HeapInuse":51331072,"HeapReleased":178249728, "HeapObjects":118290,"GCSys":8912896, "LastGC":1631588008328737479,"PauseTotalNs":67273393686, "PauseNs":[], "NumGC":36933} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:55:00.585-07:00 [Info] memstats {"Alloc":48439576, "TotalAlloc":1854802316424, "Sys":283252984, "Lookups":0, "Mallocs":157193378,"Frees":157056314, "HeapAlloc":48439576, "HeapSys":264011776, "HeapIdle":212074496, "HeapInuse":51937280,"HeapReleased":178249728, "HeapObjects":137064,"GCSys":8912896, "LastGC":1631588008328737479,"PauseTotalNs":67273393686, "PauseNs":[], "NumGC":36933} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:55:27 PDT 2021 RSS = 1861632 2021-09-13T19:55:30.614-07:00 [Info] memstats {"Alloc":44255624, "TotalAlloc":1854803867088, "Sys":283252984, "Lookups":0, "Mallocs":157212206,"Frees":157125571, "HeapAlloc":44255624, "HeapSys":264011776, "HeapIdle":213458944, "HeapInuse":50552832,"HeapReleased":178249728, "HeapObjects":86635,"GCSys":8912896, "LastGC":1631588128368099644,"PauseTotalNs":67273854375, "PauseNs":[460689], "NumGC":36934} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:56:00.650-07:00 [Info] memstats {"Alloc":45807408, "TotalAlloc":1854805418872, "Sys":283252984, "Lookups":0, "Mallocs":157230986,"Frees":157125580, "HeapAlloc":45807408, "HeapSys":264011776, "HeapIdle":212819968, "HeapInuse":51191808,"HeapReleased":178249728, "HeapObjects":105406,"GCSys":8912896, "LastGC":1631588128368099644,"PauseTotalNs":67273854375, "PauseNs":[], "NumGC":36934} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:56:27 PDT 2021 RSS = 1861632 2021-09-13 19:56:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.33800, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4497938146, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886171648, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1197959558, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672822272, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }2021-09-13T19:56:30.685-07:00 [Info] memstats {"Alloc":47429768, "TotalAlloc":1854807041232, "Sys":283252984, "Lookups":0, "Mallocs":157250028,"Frees":157125627, "HeapAlloc":47429768, "HeapSys":264011776, "HeapIdle":212140032, "HeapInuse":51871744,"HeapReleased":178249728, "HeapObjects":124401,"GCSys":8912896, "LastGC":1631588128368099644,"PauseTotalNs":67273854375, "PauseNs":[], "NumGC":36934} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:57:00.711-07:00 [Info] memstats {"Alloc":48966008, "TotalAlloc":1854808577472, "Sys":283252984, "Lookups":0, "Mallocs":157268807,"Frees":157125636, "HeapAlloc":48966008, "HeapSys":264011776, "HeapIdle":211582976, "HeapInuse":52428800,"HeapReleased":178249728, "HeapObjects":143171,"GCSys":8912896, "LastGC":1631588128368099644,"PauseTotalNs":67273854375, "PauseNs":[], "NumGC":36934} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:57:27 PDT 2021 RSS = 1861632 2021-09-13T19:57:30.741-07:00 [Info] memstats {"Alloc":44736264, "TotalAlloc":1854810087160, "Sys":283252984, "Lookups":0, "Mallocs":157287623,"Frees":157195032, "HeapAlloc":44736264, "HeapSys":264011776, "HeapIdle":212959232, "HeapInuse":51052544,"HeapReleased":178249728, "HeapObjects":92591,"GCSys":8912896, "LastGC":1631588248407536256,"PauseTotalNs":67274256117, "PauseNs":[401742], "NumGC":36935} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:58:00.779-07:00 [Info] memstats {"Alloc":46261456, "TotalAlloc":1854811612352, "Sys":283252984, "Lookups":0, "Mallocs":157306411,"Frees":157195041, "HeapAlloc":46261456, "HeapSys":264011776, "HeapIdle":212410368, "HeapInuse":51601408,"HeapReleased":178249728, "HeapObjects":111370,"GCSys":8912896, "LastGC":1631588248407536256,"PauseTotalNs":67274256117, "PauseNs":[], "NumGC":36935} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:58:27 PDT 2021 RSS = 1861632 2021-09-13T19:58:30.814-07:00 [Info] memstats {"Alloc":47805240, "TotalAlloc":1854813156136, "Sys":283252984, "Lookups":0, "Mallocs":157325228,"Frees":157195050, "HeapAlloc":47805240, "HeapSys":264011776, "HeapIdle":211845120, "HeapInuse":52166656,"HeapReleased":178249728, "HeapObjects":130178,"GCSys":8912896, "LastGC":1631588248407536256,"PauseTotalNs":67274256117, "PauseNs":[], "NumGC":36935} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T19:59:00.841-07:00 [Info] memstats {"Alloc":49348808, "TotalAlloc":1854814699704, "Sys":283252984, "Lookups":0, "Mallocs":157344008,"Frees":157195059, "HeapAlloc":49348808, "HeapSys":264011776, "HeapIdle":211271680, "HeapInuse":52740096,"HeapReleased":178249728, "HeapObjects":148949,"GCSys":8912896, "LastGC":1631588248407536256,"PauseTotalNs":67274256117, "PauseNs":[], "NumGC":36935} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 19:59:28 PDT 2021 RSS = 1861632 2021-09-13T19:59:30.869-07:00 [Info] memstats {"Alloc":45208888, "TotalAlloc":1854816220112, "Sys":283252984, "Lookups":0, "Mallocs":157362825,"Frees":157264295, "HeapAlloc":45208888, "HeapSys":264011776, "HeapIdle":212434944, "HeapInuse":51576832,"HeapReleased":178249728, "HeapObjects":98530,"GCSys":8912896, "LastGC":1631588368450986711,"PauseTotalNs":67274572934, "PauseNs":[316817], "NumGC":36936} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:00:00.901-07:00 [Info] memstats {"Alloc":46708752, "TotalAlloc":1854817719976, "Sys":283252984, "Lookups":0, "Mallocs":157381621,"Frees":157264304, "HeapAlloc":46708752, "HeapSys":264011776, "HeapIdle":211927040, "HeapInuse":52084736,"HeapReleased":178249728, "HeapObjects":117317,"GCSys":8912896, "LastGC":1631588368450986711,"PauseTotalNs":67274572934, "PauseNs":[], "NumGC":36936} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:00:28 PDT 2021 RSS = 1861632 2021-09-13T20:00:30.927-07:00 [Info] memstats {"Alloc":50252728, "TotalAlloc":1854821263952, "Sys":283252984, "Lookups":0, "Mallocs":157403907,"Frees":157264923, "HeapAlloc":50252728, "HeapSys":264011776, "HeapIdle":209125376, "HeapInuse":54886400,"HeapReleased":178249728, "HeapObjects":138984,"GCSys":8912896, "LastGC":1631588368450986711,"PauseTotalNs":67274572934, "PauseNs":[], "NumGC":36936} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:01:00.953-07:00 [Info] memstats {"Alloc":51778472, "TotalAlloc":1854822789696, "Sys":283252984, "Lookups":0, "Mallocs":157422695,"Frees":157264932, "HeapAlloc":51778472, "HeapSys":264011776, "HeapIdle":208576512, "HeapInuse":55435264,"HeapReleased":178249728, "HeapObjects":157763,"GCSys":8912896, "LastGC":1631588368450986711,"PauseTotalNs":67274572934, "PauseNs":[], "NumGC":36936} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:01:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.43800, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4497938146, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886171648, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1197959558, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672822272, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 20:01:28 PDT 2021 RSS = 1861632 2021-09-13T20:01:30.977-07:00 [Info] memstats {"Alloc":45710752, "TotalAlloc":1854824399976, "Sys":283252984, "Lookups":0, "Mallocs":157441732,"Frees":157337223, "HeapAlloc":45710752, "HeapSys":264011776, "HeapIdle":211894272, "HeapInuse":52117504,"HeapReleased":178249728, "HeapObjects":104509,"GCSys":8912896, "LastGC":1631588488505023018,"PauseTotalNs":67274990524, "PauseNs":[417590], "NumGC":36937} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:01:40 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 20:01:44 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 20:01:44 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 20:01:47 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T20:02:01.011-07:00 [Info] memstats {"Alloc":49199400, "TotalAlloc":1854827888624, "Sys":283252984, "Lookups":0, "Mallocs":157461963,"Frees":157337279, "HeapAlloc":49199400, "HeapSys":264011776, "HeapIdle":209379328, "HeapInuse":54632448,"HeapReleased":178249728, "HeapObjects":124684,"GCSys":8912896, "LastGC":1631588488505023018,"PauseTotalNs":67274990524, "PauseNs":[], "NumGC":36937} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:02:28 PDT 2021 RSS = 1861632 2021-09-13T20:02:31.042-07:00 [Info] memstats {"Alloc":50734688, "TotalAlloc":1854829423912, "Sys":283252984, "Lookups":0, "Mallocs":157480788,"Frees":157337288, "HeapAlloc":50734688, "HeapSys":264011776, "HeapIdle":208830464, "HeapInuse":55181312,"HeapReleased":178249728, "HeapObjects":143500,"GCSys":8912896, "LastGC":1631588488505023018,"PauseTotalNs":67274990524, "PauseNs":[], "NumGC":36937} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:03:01.079-07:00 [Info] memstats {"Alloc":52279904, "TotalAlloc":1854830969128, "Sys":283252984, "Lookups":0, "Mallocs":157499575,"Frees":157337297, "HeapAlloc":52279904, "HeapSys":264011776, "HeapIdle":208265216, "HeapInuse":55746560,"HeapReleased":178249728, "HeapObjects":162278,"GCSys":8912896, "LastGC":1631588488505023018,"PauseTotalNs":67274990524, "PauseNs":[], "NumGC":36937} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:03:28 PDT 2021 RSS = 1861632 2021-09-13T20:03:31.113-07:00 [Info] memstats {"Alloc":46158232, "TotalAlloc":1854832541760, "Sys":283252984, "Lookups":0, "Mallocs":157518406,"Frees":157407952, "HeapAlloc":46158232, "HeapSys":264011776, "HeapIdle":211542016, "HeapInuse":52469760,"HeapReleased":178249728, "HeapObjects":110454,"GCSys":8912896, "LastGC":1631588608550845012,"PauseTotalNs":67275401338, "PauseNs":[410814], "NumGC":36938} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:04:01.139-07:00 [Info] memstats {"Alloc":47707664, "TotalAlloc":1854834091192, "Sys":283252984, "Lookups":0, "Mallocs":157537186,"Frees":157407961, "HeapAlloc":47707664, "HeapSys":264011776, "HeapIdle":210944000, "HeapInuse":53067776,"HeapReleased":178249728, "HeapObjects":129225,"GCSys":8912896, "LastGC":1631588608550845012,"PauseTotalNs":67275401338, "PauseNs":[], "NumGC":36938} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:04:28 PDT 2021 RSS = 1861632 2021-09-13T20:04:31.166-07:00 [Info] memstats {"Alloc":49248328, "TotalAlloc":1854835631856, "Sys":283252984, "Lookups":0, "Mallocs":157556008,"Frees":157407970, "HeapAlloc":49248328, "HeapSys":264011776, "HeapIdle":210370560, "HeapInuse":53641216,"HeapReleased":178249728, "HeapObjects":148038,"GCSys":8912896, "LastGC":1631588608550845012,"PauseTotalNs":67275401338, "PauseNs":[], "NumGC":36938} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:05:01.198-07:00 [Info] memstats {"Alloc":50800712, "TotalAlloc":1854837184240, "Sys":283252984, "Lookups":0, "Mallocs":157574798,"Frees":157407979, "HeapAlloc":50800712, "HeapSys":264011776, "HeapIdle":209862656, "HeapInuse":54149120,"HeapReleased":178249728, "HeapObjects":166819,"GCSys":8912896, "LastGC":1631588608550845012,"PauseTotalNs":67275401338, "PauseNs":[], "NumGC":36938} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:05:28 PDT 2021 RSS = 1861632 2021-09-13T20:05:31.220-07:00 [Info] memstats {"Alloc":46633048, "TotalAlloc":1854838746408, "Sys":283252984, "Lookups":0, "Mallocs":157593615,"Frees":157477230, "HeapAlloc":46633048, "HeapSys":264011776, "HeapIdle":211050496, "HeapInuse":52961280,"HeapReleased":178249728, "HeapObjects":116385,"GCSys":8912896, "LastGC":1631588728604109155,"PauseTotalNs":67275808874, "PauseNs":[407536], "NumGC":36939} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:06:01.245-07:00 [Info] memstats {"Alloc":48200720, "TotalAlloc":1854840314080, "Sys":283252984, "Lookups":0, "Mallocs":157612397,"Frees":157477239, "HeapAlloc":48200720, "HeapSys":264011776, "HeapIdle":210534400, "HeapInuse":53477376,"HeapReleased":178249728, "HeapObjects":135158,"GCSys":8912896, "LastGC":1631588728604109155,"PauseTotalNs":67275808874, "PauseNs":[], "NumGC":36939} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:06:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.20400, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498003682, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886237184, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1197992326, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672855040, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 20:06:28 PDT 2021 RSS = 1861632 2021-09-13T20:06:31.272-07:00 [Info] memstats {"Alloc":49834632, "TotalAlloc":1854841947992, "Sys":283252984, "Lookups":0, "Mallocs":157631448,"Frees":157477286, "HeapAlloc":49834632, "HeapSys":264011776, "HeapIdle":209747968, "HeapInuse":54263808,"HeapReleased":178249728, "HeapObjects":154162,"GCSys":8912896, "LastGC":1631588728604109155,"PauseTotalNs":67275808874, "PauseNs":[], "NumGC":36939} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:07:01.299-07:00 [Info] memstats {"Alloc":51406008, "TotalAlloc":1854843519368, "Sys":283252984, "Lookups":0, "Mallocs":157650218,"Frees":157477295, "HeapAlloc":51406008, "HeapSys":264011776, "HeapIdle":209166336, "HeapInuse":54845440,"HeapReleased":178249728, "HeapObjects":172923,"GCSys":8912896, "LastGC":1631588728604109155,"PauseTotalNs":67275808874, "PauseNs":[], "NumGC":36939} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:07:28 PDT 2021 RSS = 1861632 2021-09-13T20:07:31.325-07:00 [Info] memstats {"Alloc":47115256, "TotalAlloc":1854845096384, "Sys":283252984, "Lookups":0, "Mallocs":157669048,"Frees":157546719, "HeapAlloc":47115256, "HeapSys":264011776, "HeapIdle":210567168, "HeapInuse":53444608,"HeapReleased":178249728, "HeapObjects":122329,"GCSys":8912896, "LastGC":1631588848670957683,"PauseTotalNs":67276221078, "PauseNs":[412204], "NumGC":36940} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:08:01.352-07:00 [Info] memstats {"Alloc":48687856, "TotalAlloc":1854846668984, "Sys":283252984, "Lookups":0, "Mallocs":157687821,"Frees":157546728, "HeapAlloc":48687856, "HeapSys":264011776, "HeapIdle":209887232, "HeapInuse":54124544,"HeapReleased":178249728, "HeapObjects":141093,"GCSys":8912896, "LastGC":1631588848670957683,"PauseTotalNs":67276221078, "PauseNs":[], "NumGC":36940} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:08:28 PDT 2021 RSS = 1861632 2021-09-13T20:08:31.380-07:00 [Info] memstats {"Alloc":50293368, "TotalAlloc":1854848274496, "Sys":283252984, "Lookups":0, "Mallocs":157706651,"Frees":157546737, "HeapAlloc":50293368, "HeapSys":264011776, "HeapIdle":209289216, "HeapInuse":54722560,"HeapReleased":178249728, "HeapObjects":159914,"GCSys":8912896, "LastGC":1631588848670957683,"PauseTotalNs":67276221078, "PauseNs":[], "NumGC":36940} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:09:01.409-07:00 [Info] memstats {"Alloc":51893688, "TotalAlloc":1854849874816, "Sys":283252984, "Lookups":0, "Mallocs":157725430,"Frees":157546746, "HeapAlloc":51893688, "HeapSys":264011776, "HeapIdle":208699392, "HeapInuse":55312384,"HeapReleased":178249728, "HeapObjects":178684,"GCSys":8912896, "LastGC":1631588848670957683,"PauseTotalNs":67276221078, "PauseNs":[], "NumGC":36940} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:09:28 PDT 2021 RSS = 1861632 2021-09-13T20:09:31.446-07:00 [Info] memstats {"Alloc":47584248, "TotalAlloc":1854851465288, "Sys":283252984, "Lookups":0, "Mallocs":157744264,"Frees":157615993, "HeapAlloc":47584248, "HeapSys":264011776, "HeapIdle":210083840, "HeapInuse":53927936,"HeapReleased":178249728, "HeapObjects":128271,"GCSys":8912896, "LastGC":1631588968720756416,"PauseTotalNs":67276524809, "PauseNs":[303731], "NumGC":36941} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:10:01.484-07:00 [Info] memstats {"Alloc":49176128, "TotalAlloc":1854853057168, "Sys":283252984, "Lookups":0, "Mallocs":157763048,"Frees":157616002, "HeapAlloc":49176128, "HeapSys":264011776, "HeapIdle":209485824, "HeapInuse":54525952,"HeapReleased":178249728, "HeapObjects":147046,"GCSys":8912896, "LastGC":1631588968720756416,"PauseTotalNs":67276524809, "PauseNs":[], "NumGC":36941} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:10:28 PDT 2021 RSS = 1861632 2021-09-13T20:10:31.512-07:00 [Info] memstats {"Alloc":52802688, "TotalAlloc":1854856683728, "Sys":283252984, "Lookups":0, "Mallocs":157785346,"Frees":157616622, "HeapAlloc":52802688, "HeapSys":264011776, "HeapIdle":206610432, "HeapInuse":57401344,"HeapReleased":178249728, "HeapObjects":168724,"GCSys":8912896, "LastGC":1631588968720756416,"PauseTotalNs":67276524809, "PauseNs":[], "NumGC":36941} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:11:01.540-07:00 [Info] memstats {"Alloc":54374656, "TotalAlloc":1854858255696, "Sys":283252984, "Lookups":0, "Mallocs":157804125,"Frees":157616631, "HeapAlloc":54374656, "HeapSys":264011776, "HeapIdle":206102528, "HeapInuse":57909248,"HeapReleased":178249728, "HeapObjects":187494,"GCSys":8912896, "LastGC":1631588968720756416,"PauseTotalNs":67276524809, "PauseNs":[], "NumGC":36941} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:11:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.22635, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498003682, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886237184, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1197992326, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672855040, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 20:11:28 PDT 2021 RSS = 1861632 2021-09-13T20:11:31.566-07:00 [Info] memstats {"Alloc":48087824, "TotalAlloc":1854859891080, "Sys":283252984, "Lookups":0, "Mallocs":157823164,"Frees":157688938, "HeapAlloc":48087824, "HeapSys":264011776, "HeapIdle":209534976, "HeapInuse":54476800,"HeapReleased":178249728, "HeapObjects":134226,"GCSys":8912896, "LastGC":1631589088780872765,"PauseTotalNs":67276960199, "PauseNs":[435390], "NumGC":36942} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:11:40 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 20:11:44 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 20:11:45 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 20:11:47 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T20:12:01.592-07:00 [Info] memstats {"Alloc":51597544, "TotalAlloc":1854863400800, "Sys":283252984, "Lookups":0, "Mallocs":157843380,"Frees":157688994, "HeapAlloc":51597544, "HeapSys":264011776, "HeapIdle":207118336, "HeapInuse":56893440,"HeapReleased":178249728, "HeapObjects":154386,"GCSys":8912896, "LastGC":1631589088780872765,"PauseTotalNs":67276960199, "PauseNs":[], "NumGC":36942} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:12:28 PDT 2021 RSS = 1861632 2021-09-13T20:12:31.620-07:00 [Info] memstats {"Alloc":53159408, "TotalAlloc":1854864962664, "Sys":283252984, "Lookups":0, "Mallocs":157862201,"Frees":157689003, "HeapAlloc":53159408, "HeapSys":264011776, "HeapIdle":206512128, "HeapInuse":57499648,"HeapReleased":178249728, "HeapObjects":173198,"GCSys":8912896, "LastGC":1631589088780872765,"PauseTotalNs":67276960199, "PauseNs":[], "NumGC":36942} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:13:01.646-07:00 [Info] memstats {"Alloc":54678608, "TotalAlloc":1854866481864, "Sys":283252984, "Lookups":0, "Mallocs":157880970,"Frees":157689012, "HeapAlloc":54678608, "HeapSys":264011776, "HeapIdle":205946880, "HeapInuse":58064896,"HeapReleased":178249728, "HeapObjects":191958,"GCSys":8912896, "LastGC":1631589088780872765,"PauseTotalNs":67276960199, "PauseNs":[], "NumGC":36942} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:13:29 PDT 2021 RSS = 1861632 2021-09-13T20:13:31.669-07:00 [Info] memstats {"Alloc":48527528, "TotalAlloc":1854868013664, "Sys":283252984, "Lookups":0, "Mallocs":157899785,"Frees":157759660, "HeapAlloc":48527528, "HeapSys":264011776, "HeapIdle":209158144, "HeapInuse":54853632,"HeapReleased":178249728, "HeapObjects":140125,"GCSys":8912896, "LastGC":1631589208847579557,"PauseTotalNs":67277366155, "PauseNs":[405956], "NumGC":36943} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:14:01.696-07:00 [Info] memstats {"Alloc":50081792, "TotalAlloc":1854869567928, "Sys":283252984, "Lookups":0, "Mallocs":157918572,"Frees":157759669, "HeapAlloc":50081792, "HeapSys":264011776, "HeapIdle":208551936, "HeapInuse":55459840,"HeapReleased":178249728, "HeapObjects":158903,"GCSys":8912896, "LastGC":1631589208847579557,"PauseTotalNs":67277366155, "PauseNs":[], "NumGC":36943} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:14:29 PDT 2021 RSS = 1861632 2021-09-13T20:14:31.733-07:00 [Info] memstats {"Alloc":51640424, "TotalAlloc":1854871126560, "Sys":283252984, "Lookups":0, "Mallocs":157937404,"Frees":157759678, "HeapAlloc":51640424, "HeapSys":264011776, "HeapIdle":207994880, "HeapInuse":56016896,"HeapReleased":178249728, "HeapObjects":177726,"GCSys":8912896, "LastGC":1631589208847579557,"PauseTotalNs":67277366155, "PauseNs":[], "NumGC":36943} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:15:01.758-07:00 [Info] memstats {"Alloc":53150584, "TotalAlloc":1854872636720, "Sys":283252984, "Lookups":0, "Mallocs":157956180,"Frees":157759687, "HeapAlloc":53150584, "HeapSys":264011776, "HeapIdle":207437824, "HeapInuse":56573952,"HeapReleased":178249728, "HeapObjects":196493,"GCSys":8912896, "LastGC":1631589208847579557,"PauseTotalNs":67277366155, "PauseNs":[], "NumGC":36943} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:15:29 PDT 2021 RSS = 1861632 2021-09-13T20:15:31.784-07:00 [Info] memstats {"Alloc":48999736, "TotalAlloc":1854874158616, "Sys":283252984, "Lookups":0, "Mallocs":157975004,"Frees":157828947, "HeapAlloc":48999736, "HeapSys":264011776, "HeapIdle":208683008, "HeapInuse":55328768,"HeapReleased":178249728, "HeapObjects":146057,"GCSys":8912896, "LastGC":1631589328915499054,"PauseTotalNs":67277699082, "PauseNs":[332927], "NumGC":36944} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:16:01.814-07:00 [Info] memstats {"Alloc":50508928, "TotalAlloc":1854875667808, "Sys":283252984, "Lookups":0, "Mallocs":157993785,"Frees":157828956, "HeapAlloc":50508928, "HeapSys":264011776, "HeapIdle":208175104, "HeapInuse":55836672,"HeapReleased":178249728, "HeapObjects":164829,"GCSys":8912896, "LastGC":1631589328915499054,"PauseTotalNs":67277699082, "PauseNs":[], "NumGC":36944} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:16:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.38800, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498069218, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886302720, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198025094, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672887808, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 20:16:29 PDT 2021 RSS = 1861632 2021-09-13T20:16:31.839-07:00 [Info] memstats {"Alloc":52078152, "TotalAlloc":1854877237032, "Sys":283252984, "Lookups":0, "Mallocs":158012826,"Frees":157829003, "HeapAlloc":52078152, "HeapSys":264011776, "HeapIdle":207478784, "HeapInuse":56532992,"HeapReleased":178249728, "HeapObjects":183823,"GCSys":8912896, "LastGC":1631589328915499054,"PauseTotalNs":67277699082, "PauseNs":[], "NumGC":36944} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:17:01.869-07:00 [Info] memstats {"Alloc":53550920, "TotalAlloc":1854878709800, "Sys":283252984, "Lookups":0, "Mallocs":158031609,"Frees":157829012, "HeapAlloc":53550920, "HeapSys":264011776, "HeapIdle":206962688, "HeapInuse":57049088,"HeapReleased":178249728, "HeapObjects":202597,"GCSys":8912896, "LastGC":1631589328915499054,"PauseTotalNs":67277699082, "PauseNs":[], "NumGC":36944} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:17:29 PDT 2021 RSS = 1861632 2021-09-13T20:17:31.903-07:00 [Info] memstats {"Alloc":49481736, "TotalAlloc":1854880222128, "Sys":283252984, "Lookups":0, "Mallocs":158050441,"Frees":157898454, "HeapAlloc":49481736, "HeapSys":264011776, "HeapIdle":208158720, "HeapInuse":55853056,"HeapReleased":178249728, "HeapObjects":151987,"GCSys":8912896, "LastGC":1631589448992609492,"PauseTotalNs":67278075035, "PauseNs":[375953], "NumGC":36945} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:18:01.929-07:00 [Info] memstats {"Alloc":51009536, "TotalAlloc":1854881749928, "Sys":283252984, "Lookups":0, "Mallocs":158069227,"Frees":157898463, "HeapAlloc":51009536, "HeapSys":264011776, "HeapIdle":207609856, "HeapInuse":56401920,"HeapReleased":178249728, "HeapObjects":170764,"GCSys":8912896, "LastGC":1631589448992609492,"PauseTotalNs":67278075035, "PauseNs":[], "NumGC":36945} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:18:29 PDT 2021 RSS = 1861632 2021-09-13T20:18:31.967-07:00 [Info] memstats {"Alloc":52581592, "TotalAlloc":1854883321984, "Sys":283252984, "Lookups":0, "Mallocs":158088060,"Frees":157898472, "HeapAlloc":52581592, "HeapSys":264011776, "HeapIdle":206970880, "HeapInuse":57040896,"HeapReleased":178249728, "HeapObjects":189588,"GCSys":8912896, "LastGC":1631589448992609492,"PauseTotalNs":67278075035, "PauseNs":[], "NumGC":36945} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:19:01.996-07:00 [Info] memstats {"Alloc":54142168, "TotalAlloc":1854884882560, "Sys":283252984, "Lookups":0, "Mallocs":158106838,"Frees":157898481, "HeapAlloc":54142168, "HeapSys":264011776, "HeapIdle":206397440, "HeapInuse":57614336,"HeapReleased":178249728, "HeapObjects":208357,"GCSys":8912896, "LastGC":1631589448992609492,"PauseTotalNs":67278075035, "PauseNs":[], "NumGC":36945} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:19:29 PDT 2021 RSS = 1861632 2021-09-13T20:19:32.023-07:00 [Info] memstats {"Alloc":49954056, "TotalAlloc":1854886453976, "Sys":283252984, "Lookups":0, "Mallocs":158125669,"Frees":157967734, "HeapAlloc":49954056, "HeapSys":264011776, "HeapIdle":207757312, "HeapInuse":56254464,"HeapReleased":178249728, "HeapObjects":157935,"GCSys":8912896, "LastGC":1631589569073327694,"PauseTotalNs":67281032029, "PauseNs":[2956994], "NumGC":36946} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:20:02.050-07:00 [Info] memstats {"Alloc":51512704, "TotalAlloc":1854888012624, "Sys":283252984, "Lookups":0, "Mallocs":158144450,"Frees":157967743, "HeapAlloc":51512704, "HeapSys":264011776, "HeapIdle":207142912, "HeapInuse":56868864,"HeapReleased":178249728, "HeapObjects":176707,"GCSys":8912896, "LastGC":1631589569073327694,"PauseTotalNs":67281032029, "PauseNs":[], "NumGC":36946} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:20:29 PDT 2021 RSS = 1861632 2021-09-13T20:20:32.078-07:00 [Info] memstats {"Alloc":55107648, "TotalAlloc":1854891607568, "Sys":283252984, "Lookups":0, "Mallocs":158166746,"Frees":157968363, "HeapAlloc":55107648, "HeapSys":264011776, "HeapIdle":204365824, "HeapInuse":59645952,"HeapReleased":178249728, "HeapObjects":198383,"GCSys":8912896, "LastGC":1631589569073327694,"PauseTotalNs":67281032029, "PauseNs":[], "NumGC":36946} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:21:02.107-07:00 [Info] memstats {"Alloc":56655920, "TotalAlloc":1854893155840, "Sys":283252984, "Lookups":0, "Mallocs":158185525,"Frees":157968372, "HeapAlloc":56655920, "HeapSys":264011776, "HeapIdle":203735040, "HeapInuse":60276736,"HeapReleased":178249728, "HeapObjects":217153,"GCSys":8912896, "LastGC":1631589569073327694,"PauseTotalNs":67281032029, "PauseNs":[], "NumGC":36946} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:21:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.30766, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498069218, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886302720, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198025094, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672887808, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 20:21:29 PDT 2021 RSS = 1861632 2021-09-13T20:21:32.134-07:00 [Info] memstats {"Alloc":50457696, "TotalAlloc":1854894789608, "Sys":283252984, "Lookups":0, "Mallocs":158204556,"Frees":158040694, "HeapAlloc":50457696, "HeapSys":264011776, "HeapIdle":207036416, "HeapInuse":56975360,"HeapReleased":178249728, "HeapObjects":163862,"GCSys":8912896, "LastGC":1631589689146420328,"PauseTotalNs":67281398476, "PauseNs":[366447], "NumGC":36947} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:21:41 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 20:21:44 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 20:21:45 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 20:21:47 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T20:22:02.161-07:00 [Info] memstats {"Alloc":54004936, "TotalAlloc":1854898336848, "Sys":283252984, "Lookups":0, "Mallocs":158224769,"Frees":158040750, "HeapAlloc":54004936, "HeapSys":264011776, "HeapIdle":204652544, "HeapInuse":59359232,"HeapReleased":178249728, "HeapObjects":184019,"GCSys":8912896, "LastGC":1631589689146420328,"PauseTotalNs":67281398476, "PauseNs":[], "NumGC":36947} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:22:29 PDT 2021 RSS = 1861632 2021-09-13T20:22:32.189-07:00 [Info] memstats {"Alloc":55625360, "TotalAlloc":1854899957272, "Sys":283252984, "Lookups":0, "Mallocs":158243598,"Frees":158040759, "HeapAlloc":55625360, "HeapSys":264011776, "HeapIdle":203972608, "HeapInuse":60039168,"HeapReleased":178249728, "HeapObjects":202839,"GCSys":8912896, "LastGC":1631589689146420328,"PauseTotalNs":67281398476, "PauseNs":[], "NumGC":36947} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:23:02.226-07:00 [Info] memstats {"Alloc":57237600, "TotalAlloc":1854901569512, "Sys":283252984, "Lookups":0, "Mallocs":158262382,"Frees":158040768, "HeapAlloc":57237600, "HeapSys":264011776, "HeapIdle":203292672, "HeapInuse":60719104,"HeapReleased":178249728, "HeapObjects":221614,"GCSys":8912896, "LastGC":1631589689146420328,"PauseTotalNs":67281398476, "PauseNs":[], "NumGC":36947} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:23:29 PDT 2021 RSS = 1861632 2021-09-13T20:23:32.252-07:00 [Info] memstats {"Alloc":50910680, "TotalAlloc":1854903195344, "Sys":283252984, "Lookups":0, "Mallocs":158281215,"Frees":158111413, "HeapAlloc":50910680, "HeapSys":264011776, "HeapIdle":206684160, "HeapInuse":57327616,"HeapReleased":178249728, "HeapObjects":169802,"GCSys":8912896, "LastGC":1631589809213984365,"PauseTotalNs":67281755118, "PauseNs":[356642], "NumGC":36948} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:24:02.277-07:00 [Info] memstats {"Alloc":52548288, "TotalAlloc":1854904832952, "Sys":283252984, "Lookups":0, "Mallocs":158300006,"Frees":158111422, "HeapAlloc":52548288, "HeapSys":264011776, "HeapIdle":206151680, "HeapInuse":57860096,"HeapReleased":178249728, "HeapObjects":188584,"GCSys":8912896, "LastGC":1631589809213984365,"PauseTotalNs":67281755118, "PauseNs":[], "NumGC":36948} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:24:29 PDT 2021 RSS = 1861632 2021-09-13T20:24:32.305-07:00 [Info] memstats {"Alloc":54198840, "TotalAlloc":1854906483504, "Sys":283252984, "Lookups":0, "Mallocs":158318826,"Frees":158111431, "HeapAlloc":54198840, "HeapSys":264011776, "HeapIdle":205365248, "HeapInuse":58646528,"HeapReleased":178249728, "HeapObjects":207395,"GCSys":8912896, "LastGC":1631589809213984365,"PauseTotalNs":67281755118, "PauseNs":[], "NumGC":36948} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:25:02.334-07:00 [Info] memstats {"Alloc":55847192, "TotalAlloc":1854908131856, "Sys":283252984, "Lookups":0, "Mallocs":158337608,"Frees":158111440, "HeapAlloc":55847192, "HeapSys":264011776, "HeapIdle":204660736, "HeapInuse":59351040,"HeapReleased":178249728, "HeapObjects":226168,"GCSys":8912896, "LastGC":1631589809213984365,"PauseTotalNs":67281755118, "PauseNs":[], "NumGC":36948} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:25:29 PDT 2021 RSS = 1861632 2021-09-13T20:25:32.363-07:00 [Info] memstats {"Alloc":51380968, "TotalAlloc":1854909767480, "Sys":283252984, "Lookups":0, "Mallocs":158356433,"Frees":158180697, "HeapAlloc":51380968, "HeapSys":264011776, "HeapIdle":206290944, "HeapInuse":57720832,"HeapReleased":178249728, "HeapObjects":175736,"GCSys":8916992, "LastGC":1631589929282916173,"PauseTotalNs":67282236404, "PauseNs":[481286], "NumGC":36949} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:26:02.389-07:00 [Info] memstats {"Alloc":52989184, "TotalAlloc":1854911375696, "Sys":283252984, "Lookups":0, "Mallocs":158375215,"Frees":158180706, "HeapAlloc":52989184, "HeapSys":264011776, "HeapIdle":205578240, "HeapInuse":58433536,"HeapReleased":178249728, "HeapObjects":194509,"GCSys":8916992, "LastGC":1631589929282916173,"PauseTotalNs":67282236404, "PauseNs":[], "NumGC":36949} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:26:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.21600, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498134754, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886368256, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198057862, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672920576, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 20:26:29 PDT 2021 RSS = 1861632 2021-09-13T20:26:32.420-07:00 [Info] memstats {"Alloc":54680920, "TotalAlloc":1854913067432, "Sys":283252984, "Lookups":0, "Mallocs":158394267,"Frees":158180753, "HeapAlloc":54680920, "HeapSys":264011776, "HeapIdle":204914688, "HeapInuse":59097088,"HeapReleased":178249728, "HeapObjects":213514,"GCSys":8916992, "LastGC":1631589929282916173,"PauseTotalNs":67282236404, "PauseNs":[], "NumGC":36949} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:27:02.449-07:00 [Info] memstats {"Alloc":56323832, "TotalAlloc":1854914710344, "Sys":283252984, "Lookups":0, "Mallocs":158413044,"Frees":158180762, "HeapAlloc":56323832, "HeapSys":264011776, "HeapIdle":204218368, "HeapInuse":59793408,"HeapReleased":178249728, "HeapObjects":232282,"GCSys":8916992, "LastGC":1631589929282916173,"PauseTotalNs":67282236404, "PauseNs":[], "NumGC":36949} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:27:29 PDT 2021 RSS = 1861632 2021-09-13T20:27:32.475-07:00 [Info] memstats {"Alloc":51866568, "TotalAlloc":1854916355760, "Sys":283252984, "Lookups":0, "Mallocs":158431859,"Frees":158250189, "HeapAlloc":51866568, "HeapSys":264011776, "HeapIdle":205692928, "HeapInuse":58318848,"HeapReleased":178249728, "HeapObjects":181670,"GCSys":8921088, "LastGC":1631590049359395997,"PauseTotalNs":67282692711, "PauseNs":[456307], "NumGC":36950} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:28:02.502-07:00 [Info] memstats {"Alloc":53525664, "TotalAlloc":1854918014856, "Sys":283252984, "Lookups":0, "Mallocs":158450640,"Frees":158250198, "HeapAlloc":53525664, "HeapSys":264011776, "HeapIdle":205070336, "HeapInuse":58941440,"HeapReleased":178249728, "HeapObjects":200442,"GCSys":8921088, "LastGC":1631590049359395997,"PauseTotalNs":67282692711, "PauseNs":[], "NumGC":36950} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:28:29 PDT 2021 RSS = 1861632 2021-09-13T20:28:32.528-07:00 [Info] memstats {"Alloc":55193576, "TotalAlloc":1854919682768, "Sys":283252984, "Lookups":0, "Mallocs":158469464,"Frees":158250207, "HeapAlloc":55193576, "HeapSys":264011776, "HeapIdle":204398592, "HeapInuse":59613184,"HeapReleased":178249728, "HeapObjects":219257,"GCSys":8921088, "LastGC":1631590049359395997,"PauseTotalNs":67282692711, "PauseNs":[], "NumGC":36950} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:29:02.560-07:00 [Info] memstats {"Alloc":56861032, "TotalAlloc":1854921350224, "Sys":283252984, "Lookups":0, "Mallocs":158488244,"Frees":158250216, "HeapAlloc":56861032, "HeapSys":264011776, "HeapIdle":203628544, "HeapInuse":60383232,"HeapReleased":178249728, "HeapObjects":238028,"GCSys":8921088, "LastGC":1631590049359395997,"PauseTotalNs":67282692711, "PauseNs":[], "NumGC":36950} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:29:30 PDT 2021 RSS = 1861632 2021-09-13T20:29:32.591-07:00 [Info] memstats {"Alloc":52339960, "TotalAlloc":1854923046600, "Sys":283252984, "Lookups":0, "Mallocs":158507073,"Frees":158319471, "HeapAlloc":52339960, "HeapSys":264011776, "HeapIdle":205144064, "HeapInuse":58867712,"HeapReleased":178249728, "HeapObjects":187602,"GCSys":8921088, "LastGC":1631590169428807600,"PauseTotalNs":67283113648, "PauseNs":[420937], "NumGC":36951} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:30:02.619-07:00 [Info] memstats {"Alloc":54049344, "TotalAlloc":1854924755984, "Sys":283252984, "Lookups":0, "Mallocs":158525863,"Frees":158319480, "HeapAlloc":54049344, "HeapSys":264011776, "HeapIdle":204578816, "HeapInuse":59432960,"HeapReleased":178249728, "HeapObjects":206383,"GCSys":8921088, "LastGC":1631590169428807600,"PauseTotalNs":67283113648, "PauseNs":[], "NumGC":36951} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:30:30 PDT 2021 RSS = 1861632 2021-09-13T20:30:32.648-07:00 [Info] memstats {"Alloc":57792408, "TotalAlloc":1854928499048, "Sys":283252984, "Lookups":0, "Mallocs":158548169,"Frees":158320103, "HeapAlloc":57792408, "HeapSys":264011776, "HeapIdle":201580544, "HeapInuse":62431232,"HeapReleased":178249728, "HeapObjects":228066,"GCSys":8921088, "LastGC":1631590169428807600,"PauseTotalNs":67283113648, "PauseNs":[], "NumGC":36951} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:31:02.677-07:00 [Info] memstats {"Alloc":59475384, "TotalAlloc":1854930182024, "Sys":283252984, "Lookups":0, "Mallocs":158566957,"Frees":158320112, "HeapAlloc":59475384, "HeapSys":264011776, "HeapIdle":200777728, "HeapInuse":63234048,"HeapReleased":178249728, "HeapObjects":246845,"GCSys":8921088, "LastGC":1631590169428807600,"PauseTotalNs":67283113648, "PauseNs":[], "NumGC":36951} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:31:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.36600, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498134754, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886368256, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198057862, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672920576, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 20:31:30 PDT 2021 RSS = 1861632 2021-09-13T20:31:32.705-07:00 [Info] memstats {"Alloc":52839920, "TotalAlloc":1854931931280, "Sys":283252984, "Lookups":0, "Mallocs":158586000,"Frees":158392453, "HeapAlloc":52839920, "HeapSys":264011776, "HeapIdle":204701696, "HeapInuse":59310080,"HeapReleased":178249728, "HeapObjects":193547,"GCSys":8921088, "LastGC":1631590289522590093,"PauseTotalNs":67283572157, "PauseNs":[458509], "NumGC":36952} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:31:41 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 20:31:45 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 20:31:45 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 20:31:47 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T20:32:02.734-07:00 [Info] memstats {"Alloc":56444152, "TotalAlloc":1854935535512, "Sys":283252984, "Lookups":0, "Mallocs":158606225,"Frees":158392509, "HeapAlloc":56444152, "HeapSys":264011776, "HeapIdle":202203136, "HeapInuse":61808640,"HeapReleased":178249728, "HeapObjects":213716,"GCSys":8921088, "LastGC":1631590289522590093,"PauseTotalNs":67283572157, "PauseNs":[], "NumGC":36952} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:32:30 PDT 2021 RSS = 1861632 2021-09-13T20:32:32.766-07:00 [Info] memstats {"Alloc":58111728, "TotalAlloc":1854937203088, "Sys":283252984, "Lookups":0, "Mallocs":158625055,"Frees":158392518, "HeapAlloc":58111728, "HeapSys":264011776, "HeapIdle":201457664, "HeapInuse":62554112,"HeapReleased":178249728, "HeapObjects":232537,"GCSys":8921088, "LastGC":1631590289522590093,"PauseTotalNs":67283572157, "PauseNs":[], "NumGC":36952} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:33:02.794-07:00 [Info] memstats {"Alloc":59789168, "TotalAlloc":1854938880528, "Sys":283252984, "Lookups":0, "Mallocs":158643838,"Frees":158392527, "HeapAlloc":59789168, "HeapSys":264011776, "HeapIdle":200728576, "HeapInuse":63283200,"HeapReleased":178249728, "HeapObjects":251311,"GCSys":8921088, "LastGC":1631590289522590093,"PauseTotalNs":67283572157, "PauseNs":[], "NumGC":36952} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:33:30 PDT 2021 RSS = 1861632 2021-09-13T20:33:32.820-07:00 [Info] memstats {"Alloc":53287944, "TotalAlloc":1854940570040, "Sys":283252984, "Lookups":0, "Mallocs":158662664,"Frees":158463201, "HeapAlloc":53287944, "HeapSys":264011776, "HeapIdle":204169216, "HeapInuse":59842560,"HeapReleased":178249728, "HeapObjects":199463,"GCSys":8921088, "LastGC":1631590409601363190,"PauseTotalNs":67283956367, "PauseNs":[384210], "NumGC":36953} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:34:02.846-07:00 [Info] memstats {"Alloc":54973392, "TotalAlloc":1854942255488, "Sys":283252984, "Lookups":0, "Mallocs":158681446,"Frees":158463210, "HeapAlloc":54973392, "HeapSys":264011776, "HeapIdle":203603968, "HeapInuse":60407808,"HeapReleased":178249728, "HeapObjects":218236,"GCSys":8921088, "LastGC":1631590409601363190,"PauseTotalNs":67283956367, "PauseNs":[], "NumGC":36953} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:34:30 PDT 2021 RSS = 1861632 2021-09-13T20:34:32.873-07:00 [Info] memstats {"Alloc":56652984, "TotalAlloc":1854943935080, "Sys":283252984, "Lookups":0, "Mallocs":158700267,"Frees":158463219, "HeapAlloc":56652984, "HeapSys":264011776, "HeapIdle":202784768, "HeapInuse":61227008,"HeapReleased":178249728, "HeapObjects":237048,"GCSys":8921088, "LastGC":1631590409601363190,"PauseTotalNs":67283956367, "PauseNs":[], "NumGC":36953} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:35:02.900-07:00 [Info] memstats {"Alloc":58325240, "TotalAlloc":1854945607336, "Sys":283252984, "Lookups":0, "Mallocs":158719042,"Frees":158463228, "HeapAlloc":58325240, "HeapSys":264011776, "HeapIdle":202129408, "HeapInuse":61882368,"HeapReleased":178249728, "HeapObjects":255814,"GCSys":8921088, "LastGC":1631590409601363190,"PauseTotalNs":67283956367, "PauseNs":[], "NumGC":36953} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:35:30 PDT 2021 RSS = 1861632 2021-09-13T20:35:32.934-07:00 [Info] memstats {"Alloc":53759320, "TotalAlloc":1854947293968, "Sys":283252984, "Lookups":0, "Mallocs":158737872,"Frees":158532489, "HeapAlloc":53759320, "HeapSys":264011776, "HeapIdle":203816960, "HeapInuse":60194816,"HeapReleased":178249728, "HeapObjects":205383,"GCSys":8921088, "LastGC":1631590529703995342,"PauseTotalNs":67284347248, "PauseNs":[390881], "NumGC":36954} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:36:02.962-07:00 [Info] memstats {"Alloc":55460928, "TotalAlloc":1854948995576, "Sys":283252984, "Lookups":0, "Mallocs":158756664,"Frees":158532498, "HeapAlloc":55460928, "HeapSys":264011776, "HeapIdle":203079680, "HeapInuse":60932096,"HeapReleased":178249728, "HeapObjects":224166,"GCSys":8921088, "LastGC":1631590529703995342,"PauseTotalNs":67284347248, "PauseNs":[], "NumGC":36954} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:36:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.42200, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498200290, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886433792, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198090630, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672953344, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 20:36:30 PDT 2021 RSS = 1861632 2021-09-13T20:36:32.988-07:00 [Info] memstats {"Alloc":57236792, "TotalAlloc":1854950771440, "Sys":283252984, "Lookups":0, "Mallocs":158775704,"Frees":158532545, "HeapAlloc":57236792, "HeapSys":264011776, "HeapIdle":202203136, "HeapInuse":61808640,"HeapReleased":178249728, "HeapObjects":243159,"GCSys":8921088, "LastGC":1631590529703995342,"PauseTotalNs":67284347248, "PauseNs":[], "NumGC":36954} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:37:03.017-07:00 [Info] memstats {"Alloc":58949432, "TotalAlloc":1854952484080, "Sys":283252984, "Lookups":0, "Mallocs":158794490,"Frees":158532554, "HeapAlloc":58949432, "HeapSys":264011776, "HeapIdle":201416704, "HeapInuse":62595072,"HeapReleased":178249728, "HeapObjects":261936,"GCSys":8921088, "LastGC":1631590529703995342,"PauseTotalNs":67284347248, "PauseNs":[], "NumGC":36954} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:37:30 PDT 2021 RSS = 1861632 2021-09-13T20:37:33.044-07:00 [Info] memstats {"Alloc":54248568, "TotalAlloc":1854954224760, "Sys":283252984, "Lookups":0, "Mallocs":158813328,"Frees":158601994, "HeapAlloc":54248568, "HeapSys":264011776, "HeapIdle":203235328, "HeapInuse":60776448,"HeapReleased":178249728, "HeapObjects":211334,"GCSys":8921088, "LastGC":1631590649792761228,"PauseTotalNs":67284756539, "PauseNs":[409291], "NumGC":36955} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:38:03.072-07:00 [Info] memstats {"Alloc":55965360, "TotalAlloc":1854955941552, "Sys":283252984, "Lookups":0, "Mallocs":158832111,"Frees":158602003, "HeapAlloc":55965360, "HeapSys":264011776, "HeapIdle":202457088, "HeapInuse":61554688,"HeapReleased":178249728, "HeapObjects":230108,"GCSys":8921088, "LastGC":1631590649792761228,"PauseTotalNs":67284756539, "PauseNs":[], "NumGC":36955} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:38:30 PDT 2021 RSS = 1861632 2021-09-13T20:38:33.101-07:00 [Info] memstats {"Alloc":57699336, "TotalAlloc":1854957675528, "Sys":283252984, "Lookups":0, "Mallocs":158850947,"Frees":158602012, "HeapAlloc":57699336, "HeapSys":264011776, "HeapIdle":201818112, "HeapInuse":62193664,"HeapReleased":178249728, "HeapObjects":248935,"GCSys":8921088, "LastGC":1631590649792761228,"PauseTotalNs":67284756539, "PauseNs":[], "NumGC":36955} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:39:03.131-07:00 [Info] memstats {"Alloc":59457464, "TotalAlloc":1854959433656, "Sys":283252984, "Lookups":0, "Mallocs":158869725,"Frees":158602021, "HeapAlloc":59457464, "HeapSys":264011776, "HeapIdle":201064448, "HeapInuse":62947328,"HeapReleased":178249728, "HeapObjects":267704,"GCSys":8921088, "LastGC":1631590649792761228,"PauseTotalNs":67284756539, "PauseNs":[], "NumGC":36955} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:39:30 PDT 2021 RSS = 1861632 2021-09-13T20:39:33.158-07:00 [Info] memstats {"Alloc":54716280, "TotalAlloc":1854961190944, "Sys":283252984, "Lookups":0, "Mallocs":158888560,"Frees":158671321, "HeapAlloc":54716280, "HeapSys":264011776, "HeapIdle":202924032, "HeapInuse":61087744,"HeapReleased":178249728, "HeapObjects":217239,"GCSys":8921088, "LastGC":1631590769891525831,"PauseTotalNs":67285136477, "PauseNs":[379938], "NumGC":36956} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:40:03.185-07:00 [Info] memstats {"Alloc":56460928, "TotalAlloc":1854962935592, "Sys":283252984, "Lookups":0, "Mallocs":158907341,"Frees":158671330, "HeapAlloc":56460928, "HeapSys":264011776, "HeapIdle":202145792, "HeapInuse":61865984,"HeapReleased":178249728, "HeapObjects":236011,"GCSys":8921088, "LastGC":1631590769891525831,"PauseTotalNs":67285136477, "PauseNs":[], "NumGC":36956} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:40:30 PDT 2021 RSS = 1861632 2021-09-13T20:40:33.213-07:00 [Info] memstats {"Alloc":60255856, "TotalAlloc":1854966730520, "Sys":283252984, "Lookups":0, "Mallocs":158929658,"Frees":158671956, "HeapAlloc":60255856, "HeapSys":264011776, "HeapIdle":199090176, "HeapInuse":64921600,"HeapReleased":178249728, "HeapObjects":257702,"GCSys":8921088, "LastGC":1631590769891525831,"PauseTotalNs":67285136477, "PauseNs":[], "NumGC":36956} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:41:03.242-07:00 [Info] memstats {"Alloc":62017216, "TotalAlloc":1854968491880, "Sys":283252984, "Lookups":0, "Mallocs":158948451,"Frees":158671965, "HeapAlloc":62017216, "HeapSys":264011776, "HeapIdle":198213632, "HeapInuse":65798144,"HeapReleased":178249728, "HeapObjects":276486,"GCSys":8921088, "LastGC":1631590769891525831,"PauseTotalNs":67285136477, "PauseNs":[], "NumGC":36956} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:41:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.44400, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498200290, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886433792, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198090630, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672953344, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 20:41:30 PDT 2021 RSS = 1861632 2021-09-13T20:41:33.276-07:00 [Info] memstats {"Alloc":55221040, "TotalAlloc":1854970340912, "Sys":283318520, "Lookups":0, "Mallocs":158967501,"Frees":158744329, "HeapAlloc":55221040, "HeapSys":264011776, "HeapIdle":202301440, "HeapInuse":61710336,"HeapReleased":178249728, "HeapObjects":223172,"GCSys":8990720, "LastGC":1631590889984366939,"PauseTotalNs":67285478712, "PauseNs":[342235], "NumGC":36957} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:41:41 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 20:41:45 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 20:41:45 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 20:41:48 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T20:42:03.302-07:00 [Info] memstats {"Alloc":58911368, "TotalAlloc":1854974031240, "Sys":283318520, "Lookups":0, "Mallocs":158987734,"Frees":158744384, "HeapAlloc":58911368, "HeapSys":264011776, "HeapIdle":199622656, "HeapInuse":64389120,"HeapReleased":178249728, "HeapObjects":243350,"GCSys":8990720, "LastGC":1631590889984366939,"PauseTotalNs":67285478712, "PauseNs":[], "NumGC":36957} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:42:30 PDT 2021 RSS = 1861632 2021-09-13T20:42:33.329-07:00 [Info] memstats {"Alloc":60651488, "TotalAlloc":1854975771360, "Sys":283318520, "Lookups":0, "Mallocs":159006570,"Frees":158744393, "HeapAlloc":60651488, "HeapSys":264011776, "HeapIdle":198868992, "HeapInuse":65142784,"HeapReleased":178249728, "HeapObjects":262177,"GCSys":8990720, "LastGC":1631590889984366939,"PauseTotalNs":67285478712, "PauseNs":[], "NumGC":36957} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:43:03.357-07:00 [Info] memstats {"Alloc":62391744, "TotalAlloc":1854977511616, "Sys":283318520, "Lookups":0, "Mallocs":159025358,"Frees":158744402, "HeapAlloc":62391744, "HeapSys":264011776, "HeapIdle":198090752, "HeapInuse":65921024,"HeapReleased":178249728, "HeapObjects":280956,"GCSys":8994816, "LastGC":1631590889984366939,"PauseTotalNs":67285478712, "PauseNs":[], "NumGC":36957} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:43:30 PDT 2021 RSS = 1861632 2021-09-13T20:43:33.384-07:00 [Info] memstats {"Alloc":55663928, "TotalAlloc":1854979258120, "Sys":283384056, "Lookups":0, "Mallocs":159044190,"Frees":158815100, "HeapAlloc":55663928, "HeapSys":264011776, "HeapIdle":201859072, "HeapInuse":62152704,"HeapReleased":178249728, "HeapObjects":229090,"GCSys":9060352, "LastGC":1631591010089271490,"PauseTotalNs":67285791579, "PauseNs":[312867], "NumGC":36958} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:44:03.411-07:00 [Info] memstats {"Alloc":57407456, "TotalAlloc":1854981001648, "Sys":283384056, "Lookups":0, "Mallocs":159062986,"Frees":158815109, "HeapAlloc":57407456, "HeapSys":264011776, "HeapIdle":201220096, "HeapInuse":62791680,"HeapReleased":178249728, "HeapObjects":247877,"GCSys":9060352, "LastGC":1631591010089271490,"PauseTotalNs":67285791579, "PauseNs":[], "NumGC":36958} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:44:30 PDT 2021 RSS = 1861632 2021-09-13T20:44:33.441-07:00 [Info] memstats {"Alloc":59123288, "TotalAlloc":1854982717480, "Sys":283384056, "Lookups":0, "Mallocs":159081817,"Frees":158815118, "HeapAlloc":59123288, "HeapSys":264011776, "HeapIdle":200515584, "HeapInuse":63496192,"HeapReleased":178249728, "HeapObjects":266699,"GCSys":9060352, "LastGC":1631591010089271490,"PauseTotalNs":67285791579, "PauseNs":[], "NumGC":36958} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:45:03.467-07:00 [Info] memstats {"Alloc":60857864, "TotalAlloc":1854984452056, "Sys":283384056, "Lookups":0, "Mallocs":159100606,"Frees":158815127, "HeapAlloc":60857864, "HeapSys":264011776, "HeapIdle":199548928, "HeapInuse":64462848,"HeapReleased":178249728, "HeapObjects":285479,"GCSys":9060352, "LastGC":1631591010089271490,"PauseTotalNs":67285791579, "PauseNs":[], "NumGC":36958} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:45:31 PDT 2021 RSS = 1861632 2021-09-13T20:45:33.493-07:00 [Info] memstats {"Alloc":56131304, "TotalAlloc":1854986180320, "Sys":283449592, "Lookups":0, "Mallocs":159119434,"Frees":158884464, "HeapAlloc":56131304, "HeapSys":264011776, "HeapIdle":201334784, "HeapInuse":62676992,"HeapReleased":178249728, "HeapObjects":234970,"GCSys":9125888, "LastGC":1631591130194944493,"PauseTotalNs":67286144183, "PauseNs":[352604], "NumGC":36959} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:46:03.520-07:00 [Info] memstats {"Alloc":57844944, "TotalAlloc":1854987893960, "Sys":283449592, "Lookups":0, "Mallocs":159138228,"Frees":158884473, "HeapAlloc":57844944, "HeapSys":264011776, "HeapIdle":200728576, "HeapInuse":63283200,"HeapReleased":178249728, "HeapObjects":253755,"GCSys":9125888, "LastGC":1631591130194944493,"PauseTotalNs":67286144183, "PauseNs":[], "NumGC":36959} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:46:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.34800, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498265826, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886499328, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198123398, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672986112, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 20:46:31 PDT 2021 RSS = 1861632 2021-09-13T20:46:33.549-07:00 [Info] memstats {"Alloc":59593512, "TotalAlloc":1854989642528, "Sys":283449592, "Lookups":0, "Mallocs":159157277,"Frees":158884520, "HeapAlloc":59593512, "HeapSys":264011776, "HeapIdle":199876608, "HeapInuse":64135168,"HeapReleased":178249728, "HeapObjects":272757,"GCSys":9125888, "LastGC":1631591130194944493,"PauseTotalNs":67286144183, "PauseNs":[], "NumGC":36959} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:47:03.576-07:00 [Info] memstats {"Alloc":61260472, "TotalAlloc":1854991309488, "Sys":283449592, "Lookups":0, "Mallocs":159176065,"Frees":158884529, "HeapAlloc":61260472, "HeapSys":264011776, "HeapIdle":199163904, "HeapInuse":64847872,"HeapReleased":178249728, "HeapObjects":291536,"GCSys":9125888, "LastGC":1631591130194944493,"PauseTotalNs":67286144183, "PauseNs":[], "NumGC":36959} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:47:31 PDT 2021 RSS = 1861632 2021-09-13T20:47:33.601-07:00 [Info] memstats {"Alloc":56602888, "TotalAlloc":1854992975992, "Sys":283449592, "Lookups":0, "Mallocs":159194895,"Frees":158954027, "HeapAlloc":56602888, "HeapSys":264011776, "HeapIdle":200908800, "HeapInuse":63102976,"HeapReleased":178249728, "HeapObjects":240868,"GCSys":9125888, "LastGC":1631591250308174783,"PauseTotalNs":67286510069, "PauseNs":[365886], "NumGC":36960} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:48:03.632-07:00 [Info] memstats {"Alloc":58276848, "TotalAlloc":1854994649952, "Sys":283449592, "Lookups":0, "Mallocs":159213698,"Frees":158954036, "HeapAlloc":58276848, "HeapSys":264011776, "HeapIdle":200187904, "HeapInuse":63823872,"HeapReleased":178249728, "HeapObjects":259662,"GCSys":9125888, "LastGC":1631591250308174783,"PauseTotalNs":67286510069, "PauseNs":[], "NumGC":36960} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:48:31 PDT 2021 RSS = 1861632 2021-09-13T20:48:33.659-07:00 [Info] memstats {"Alloc":59960120, "TotalAlloc":1854996333224, "Sys":283449592, "Lookups":0, "Mallocs":159232537,"Frees":158954045, "HeapAlloc":59960120, "HeapSys":264011776, "HeapIdle":199491584, "HeapInuse":64520192,"HeapReleased":178249728, "HeapObjects":278492,"GCSys":9125888, "LastGC":1631591250308174783,"PauseTotalNs":67286510069, "PauseNs":[], "NumGC":36960} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:49:03.684-07:00 [Info] memstats {"Alloc":61637848, "TotalAlloc":1854998010952, "Sys":283449592, "Lookups":0, "Mallocs":159251345,"Frees":158954054, "HeapAlloc":61637848, "HeapSys":264011776, "HeapIdle":198762496, "HeapInuse":65249280,"HeapReleased":178249728, "HeapObjects":297291,"GCSys":9125888, "LastGC":1631591250308174783,"PauseTotalNs":67286510069, "PauseNs":[], "NumGC":36960} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:49:31 PDT 2021 RSS = 1861632 2021-09-13T20:49:33.709-07:00 [Info] memstats {"Alloc":57074568, "TotalAlloc":1854999693232, "Sys":283449592, "Lookups":0, "Mallocs":159270187,"Frees":159023381, "HeapAlloc":57074568, "HeapSys":264011776, "HeapIdle":200491008, "HeapInuse":63520768,"HeapReleased":178249728, "HeapObjects":246806,"GCSys":9125888, "LastGC":1631591370409640481,"PauseTotalNs":67286823991, "PauseNs":[313922], "NumGC":36961} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:50:03.735-07:00 [Info] memstats {"Alloc":58748880, "TotalAlloc":1855001367544, "Sys":283449592, "Lookups":0, "Mallocs":159288991,"Frees":159023390, "HeapAlloc":58748880, "HeapSys":264011776, "HeapIdle":199761920, "HeapInuse":64249856,"HeapReleased":178249728, "HeapObjects":265601,"GCSys":9125888, "LastGC":1631591370409640481,"PauseTotalNs":67286823991, "PauseNs":[], "NumGC":36961} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:50:31 PDT 2021 RSS = 1861632 2021-09-13T20:50:33.762-07:00 [Info] memstats {"Alloc":62461616, "TotalAlloc":1855005080280, "Sys":283449592, "Lookups":0, "Mallocs":159311331,"Frees":159024017, "HeapAlloc":62461616, "HeapSys":264011776, "HeapIdle":196870144, "HeapInuse":67141632,"HeapReleased":178249728, "HeapObjects":287314,"GCSys":9125888, "LastGC":1631591370409640481,"PauseTotalNs":67286823991, "PauseNs":[], "NumGC":36961} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:51:03.789-07:00 [Info] memstats {"Alloc":64115312, "TotalAlloc":1855006733976, "Sys":283449592, "Lookups":0, "Mallocs":159330118,"Frees":159024026, "HeapAlloc":64115312, "HeapSys":264011776, "HeapIdle":196182016, "HeapInuse":67829760,"HeapReleased":178249728, "HeapObjects":306092,"GCSys":9125888, "LastGC":1631591370409640481,"PauseTotalNs":67286823991, "PauseNs":[], "NumGC":36961} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:51:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.31600, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498265826, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886499328, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198123398, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86672986112, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 20:51:31 PDT 2021 RSS = 1861632 2021-09-13T20:51:33.817-07:00 [Info] memstats {"Alloc":57571152, "TotalAlloc":1855008471792, "Sys":283449592, "Lookups":0, "Mallocs":159349168,"Frees":159096469, "HeapAlloc":57571152, "HeapSys":264011776, "HeapIdle":199933952, "HeapInuse":64077824,"HeapReleased":178249728, "HeapObjects":252699,"GCSys":9125888, "LastGC":1631591490503534917,"PauseTotalNs":67287317903, "PauseNs":[493912], "NumGC":36962} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:51:41 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 20:51:45 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 20:51:45 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 20:51:48 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T20:52:03.846-07:00 [Info] memstats {"Alloc":61183608, "TotalAlloc":1855012084248, "Sys":283449592, "Lookups":0, "Mallocs":159369398,"Frees":159096524, "HeapAlloc":61183608, "HeapSys":264011776, "HeapIdle":197410816, "HeapInuse":66600960,"HeapReleased":178249728, "HeapObjects":272874,"GCSys":9125888, "LastGC":1631591490503534917,"PauseTotalNs":67287317903, "PauseNs":[], "NumGC":36962} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:52:31 PDT 2021 RSS = 1861632 2021-09-13T20:52:33.879-07:00 [Info] memstats {"Alloc":62847776, "TotalAlloc":1855013748416, "Sys":283449592, "Lookups":0, "Mallocs":159388222,"Frees":159096533, "HeapAlloc":62847776, "HeapSys":264011776, "HeapIdle":196763648, "HeapInuse":67248128,"HeapReleased":178249728, "HeapObjects":291689,"GCSys":9125888, "LastGC":1631591490503534917,"PauseTotalNs":67287317903, "PauseNs":[], "NumGC":36962} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:53:03.910-07:00 [Info] memstats {"Alloc":64518848, "TotalAlloc":1855015419488, "Sys":283449592, "Lookups":0, "Mallocs":159407015,"Frees":159096542, "HeapAlloc":64518848, "HeapSys":264011776, "HeapIdle":195977216, "HeapInuse":68034560,"HeapReleased":178249728, "HeapObjects":310473,"GCSys":9125888, "LastGC":1631591490503534917,"PauseTotalNs":67287317903, "PauseNs":[], "NumGC":36962} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:53:31 PDT 2021 RSS = 1861632 2021-09-13T20:53:33.937-07:00 [Info] memstats {"Alloc":58019816, "TotalAlloc":1855017112488, "Sys":283449592, "Lookups":0, "Mallocs":159425838,"Frees":159167220, "HeapAlloc":58019816, "HeapSys":264011776, "HeapIdle":199532544, "HeapInuse":64479232,"HeapReleased":178249728, "HeapObjects":258618,"GCSys":9125888, "LastGC":1631591610616614722,"PauseTotalNs":67287673438, "PauseNs":[355535], "NumGC":36963} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:54:03.963-07:00 [Info] memstats {"Alloc":59707760, "TotalAlloc":1855018800432, "Sys":283449592, "Lookups":0, "Mallocs":159444628,"Frees":159167229, "HeapAlloc":59707760, "HeapSys":264011776, "HeapIdle":198795264, "HeapInuse":65216512,"HeapReleased":178249728, "HeapObjects":277399,"GCSys":9125888, "LastGC":1631591610616614722,"PauseTotalNs":67287673438, "PauseNs":[], "NumGC":36963} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:54:31 PDT 2021 RSS = 1861632 2021-09-13T20:54:33.989-07:00 [Info] memstats {"Alloc":61432424, "TotalAlloc":1855020525096, "Sys":283449592, "Lookups":0, "Mallocs":159463451,"Frees":159167238, "HeapAlloc":61432424, "HeapSys":264011776, "HeapIdle":198164480, "HeapInuse":65847296,"HeapReleased":178249728, "HeapObjects":296213,"GCSys":9125888, "LastGC":1631591610616614722,"PauseTotalNs":67287673438, "PauseNs":[], "NumGC":36963} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:55:04.026-07:00 [Info] memstats {"Alloc":63158008, "TotalAlloc":1855022250680, "Sys":283449592, "Lookups":0, "Mallocs":159482259,"Frees":159167247, "HeapAlloc":63158008, "HeapSys":264011776, "HeapIdle":197238784, "HeapInuse":66772992,"HeapReleased":178249728, "HeapObjects":315012,"GCSys":9125888, "LastGC":1631591610616614722,"PauseTotalNs":67287673438, "PauseNs":[], "NumGC":36963} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:55:31 PDT 2021 RSS = 1861632 2021-09-13T20:55:34.053-07:00 [Info] memstats {"Alloc":58496520, "TotalAlloc":1855023967920, "Sys":283449592, "Lookups":0, "Mallocs":159501096,"Frees":159236555, "HeapAlloc":58496520, "HeapSys":264011776, "HeapIdle":199057408, "HeapInuse":64954368,"HeapReleased":178249728, "HeapObjects":264541,"GCSys":9125888, "LastGC":1631591730736987847,"PauseTotalNs":67288047685, "PauseNs":[374247], "NumGC":36964} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:56:04.080-07:00 [Info] memstats {"Alloc":60212448, "TotalAlloc":1855025683848, "Sys":283449592, "Lookups":0, "Mallocs":159519898,"Frees":159236564, "HeapAlloc":60212448, "HeapSys":264011776, "HeapIdle":198320128, "HeapInuse":65691648,"HeapReleased":178249728, "HeapObjects":283334,"GCSys":9125888, "LastGC":1631591730736987847,"PauseTotalNs":67288047685, "PauseNs":[], "NumGC":36964} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 20:56:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.39800, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498331362, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886564864, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198156166, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86673018880, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 20:56:31 PDT 2021 RSS = 1861632 2021-09-13T20:56:34.105-07:00 [Info] memstats {"Alloc":61984824, "TotalAlloc":1855027456224, "Sys":283449592, "Lookups":0, "Mallocs":159538943,"Frees":159236611, "HeapAlloc":61984824, "HeapSys":264011776, "HeapIdle":197345280, "HeapInuse":66666496,"HeapReleased":178249728, "HeapObjects":302332,"GCSys":9125888, "LastGC":1631591730736987847,"PauseTotalNs":67288047685, "PauseNs":[], "NumGC":36964} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:57:04.139-07:00 [Info] memstats {"Alloc":63685544, "TotalAlloc":1855029156944, "Sys":283449592, "Lookups":0, "Mallocs":159557733,"Frees":159236620, "HeapAlloc":63685544, "HeapSys":264011776, "HeapIdle":196714496, "HeapInuse":67297280,"HeapReleased":178249728, "HeapObjects":321113,"GCSys":9125888, "LastGC":1631591730736987847,"PauseTotalNs":67288047685, "PauseNs":[], "NumGC":36964} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:57:31 PDT 2021 RSS = 1861632 2021-09-13T20:57:34.171-07:00 [Info] memstats {"Alloc":58976808, "TotalAlloc":1855030872408, "Sys":283449592, "Lookups":0, "Mallocs":159576570,"Frees":159306101, "HeapAlloc":58976808, "HeapSys":264011776, "HeapIdle":198483968, "HeapInuse":65527808,"HeapReleased":178249728, "HeapObjects":270469,"GCSys":9125888, "LastGC":1631591850875701933,"PauseTotalNs":67288424017, "PauseNs":[376332], "NumGC":36965} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:58:04.197-07:00 [Info] memstats {"Alloc":60688640, "TotalAlloc":1855032584240, "Sys":283449592, "Lookups":0, "Mallocs":159595373,"Frees":159306110, "HeapAlloc":60688640, "HeapSys":264011776, "HeapIdle":197885952, "HeapInuse":66125824,"HeapReleased":178249728, "HeapObjects":289263,"GCSys":9125888, "LastGC":1631591850875701933,"PauseTotalNs":67288424017, "PauseNs":[], "NumGC":36965} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:58:31 PDT 2021 RSS = 1861632 2021-09-13T20:58:34.224-07:00 [Info] memstats {"Alloc":62398904, "TotalAlloc":1855034294504, "Sys":283449592, "Lookups":0, "Mallocs":159614211,"Frees":159306119, "HeapAlloc":62398904, "HeapSys":264011776, "HeapIdle":197050368, "HeapInuse":66961408,"HeapReleased":178249728, "HeapObjects":308092,"GCSys":9125888, "LastGC":1631591850875701933,"PauseTotalNs":67288424017, "PauseNs":[], "NumGC":36965} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T20:59:04.248-07:00 [Info] memstats {"Alloc":64074440, "TotalAlloc":1855035970040, "Sys":283449592, "Lookups":0, "Mallocs":159632993,"Frees":159306128, "HeapAlloc":64074440, "HeapSys":264011776, "HeapIdle":196321280, "HeapInuse":67690496,"HeapReleased":178249728, "HeapObjects":326865,"GCSys":9125888, "LastGC":1631591850875701933,"PauseTotalNs":67288424017, "PauseNs":[], "NumGC":36965} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 20:59:31 PDT 2021 RSS = 1861632 2021-09-13T20:59:34.275-07:00 [Info] memstats {"Alloc":59436616, "TotalAlloc":1855037662448, "Sys":283449592, "Lookups":0, "Mallocs":159651827,"Frees":159375491, "HeapAlloc":59436616, "HeapSys":264011776, "HeapIdle":198082560, "HeapInuse":65929216,"HeapReleased":178249728, "HeapObjects":276336,"GCSys":9129984, "LastGC":1631591971001738299,"PauseTotalNs":67288766159, "PauseNs":[342142], "NumGC":36966} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:00:04.302-07:00 [Info] memstats {"Alloc":61165920, "TotalAlloc":1855039391752, "Sys":283449592, "Lookups":0, "Mallocs":159670641,"Frees":159375500, "HeapAlloc":61165920, "HeapSys":264011776, "HeapIdle":197312512, "HeapInuse":66699264,"HeapReleased":178249728, "HeapObjects":295141,"GCSys":9129984, "LastGC":1631591971001738299,"PauseTotalNs":67288766159, "PauseNs":[], "NumGC":36966} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:00:31 PDT 2021 RSS = 1861632 2021-09-13T21:00:34.329-07:00 [Info] memstats {"Alloc":64942784, "TotalAlloc":1855043168616, "Sys":283449592, "Lookups":0, "Mallocs":159693004,"Frees":159376130, "HeapAlloc":64942784, "HeapSys":264011776, "HeapIdle":194265088, "HeapInuse":69746688,"HeapReleased":178249728, "HeapObjects":316874,"GCSys":9129984, "LastGC":1631591971001738299,"PauseTotalNs":67288766159, "PauseNs":[], "NumGC":36966} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:01:04.357-07:00 [Info] memstats {"Alloc":66645856, "TotalAlloc":1855044871688, "Sys":283449592, "Lookups":0, "Mallocs":159711801,"Frees":159376139, "HeapAlloc":66645856, "HeapSys":264011776, "HeapIdle":193601536, "HeapInuse":70410240,"HeapReleased":178249728, "HeapObjects":335662,"GCSys":9134080, "LastGC":1631591971001738299,"PauseTotalNs":67288766159, "PauseNs":[], "NumGC":36966} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 21:01:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.33600, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498331362, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886564864, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198156166, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86673018880, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 21:01:32 PDT 2021 RSS = 1861632 2021-09-13T21:01:34.382-07:00 [Info] memstats {"Alloc":59941248, "TotalAlloc":1855046658736, "Sys":283449592, "Lookups":0, "Mallocs":159730881,"Frees":159448626, "HeapAlloc":59941248, "HeapSys":264011776, "HeapIdle":197402624, "HeapInuse":66609152,"HeapReleased":178249728, "HeapObjects":282255,"GCSys":9134080, "LastGC":1631592091117727285,"PauseTotalNs":67289048289, "PauseNs":[282130], "NumGC":36967} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 21:01:41 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 21:01:45 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 21:01:46 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 21:01:48 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T21:02:04.408-07:00 [Info] memstats {"Alloc":63603768, "TotalAlloc":1855050321256, "Sys":283449592, "Lookups":0, "Mallocs":159751121,"Frees":159448681, "HeapAlloc":63603768, "HeapSys":264011776, "HeapIdle":194879488, "HeapInuse":69132288,"HeapReleased":178249728, "HeapObjects":302440,"GCSys":9134080, "LastGC":1631592091117727285,"PauseTotalNs":67289048289, "PauseNs":[], "NumGC":36967} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:02:32 PDT 2021 RSS = 1861632 2021-09-13T21:02:34.437-07:00 [Info] memstats {"Alloc":65337648, "TotalAlloc":1855052055136, "Sys":283449592, "Lookups":0, "Mallocs":159769984,"Frees":159448690, "HeapAlloc":65337648, "HeapSys":264011776, "HeapIdle":194035712, "HeapInuse":69976064,"HeapReleased":178249728, "HeapObjects":321294,"GCSys":9134080, "LastGC":1631592091117727285,"PauseTotalNs":67289048289, "PauseNs":[], "NumGC":36967} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:03:04.463-07:00 [Info] memstats {"Alloc":67053680, "TotalAlloc":1855053771168, "Sys":283449592, "Lookups":0, "Mallocs":159788787,"Frees":159448699, "HeapAlloc":67053680, "HeapSys":264011776, "HeapIdle":193265664, "HeapInuse":70746112,"HeapReleased":178249728, "HeapObjects":340088,"GCSys":9134080, "LastGC":1631592091117727285,"PauseTotalNs":67289048289, "PauseNs":[], "NumGC":36967} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:03:32 PDT 2021 RSS = 1861632 2021-09-13T21:03:34.489-07:00 [Info] memstats {"Alloc":60394120, "TotalAlloc":1855055493096, "Sys":283449592, "Lookups":0, "Mallocs":159807629,"Frees":159519461, "HeapAlloc":60394120, "HeapSys":264011776, "HeapIdle":196976640, "HeapInuse":67035136,"HeapReleased":178249728, "HeapObjects":288168,"GCSys":9134080, "LastGC":1631592211227592389,"PauseTotalNs":67289453077, "PauseNs":[404788], "NumGC":36968} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:04:04.520-07:00 [Info] memstats {"Alloc":62101760, "TotalAlloc":1855057200736, "Sys":283449592, "Lookups":0, "Mallocs":159826421,"Frees":159519470, "HeapAlloc":62101760, "HeapSys":264011776, "HeapIdle":196378624, "HeapInuse":67633152,"HeapReleased":178249728, "HeapObjects":306951,"GCSys":9134080, "LastGC":1631592211227592389,"PauseTotalNs":67289453077, "PauseNs":[], "NumGC":36968} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:04:32 PDT 2021 RSS = 1861632 2021-09-13T21:04:34.546-07:00 [Info] memstats {"Alloc":63802408, "TotalAlloc":1855058901384, "Sys":283449592, "Lookups":0, "Mallocs":159845262,"Frees":159519479, "HeapAlloc":63802408, "HeapSys":264011776, "HeapIdle":195698688, "HeapInuse":68313088,"HeapReleased":178249728, "HeapObjects":325783,"GCSys":9134080, "LastGC":1631592211227592389,"PauseTotalNs":67289453077, "PauseNs":[], "NumGC":36968} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:05:04.572-07:00 [Info] memstats {"Alloc":65487480, "TotalAlloc":1855060586456, "Sys":283449592, "Lookups":0, "Mallocs":159864066,"Frees":159519491, "HeapAlloc":65487480, "HeapSys":264011776, "HeapIdle":194822144, "HeapInuse":69189632,"HeapReleased":178249728, "HeapObjects":344575,"GCSys":9134080, "LastGC":1631592211227592389,"PauseTotalNs":67289453077, "PauseNs":[], "NumGC":36968} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:05:32 PDT 2021 RSS = 1861632 2021-09-13T21:05:34.595-07:00 [Info] memstats {"Alloc":60854936, "TotalAlloc":1855062283728, "Sys":283449592, "Lookups":0, "Mallocs":159882898,"Frees":159588842, "HeapAlloc":60854936, "HeapSys":264011776, "HeapIdle":196657152, "HeapInuse":67354624,"HeapReleased":178249728, "HeapObjects":294056,"GCSys":9134080, "LastGC":1631592331373230945,"PauseTotalNs":67289830758, "PauseNs":[377681], "NumGC":36969} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:06:04.622-07:00 [Info] memstats {"Alloc":62521392, "TotalAlloc":1855063950184, "Sys":283449592, "Lookups":0, "Mallocs":159901687,"Frees":159588851, "HeapAlloc":62521392, "HeapSys":264011776, "HeapIdle":196009984, "HeapInuse":68001792,"HeapReleased":178249728, "HeapObjects":312836,"GCSys":9134080, "LastGC":1631592331373230945,"PauseTotalNs":67289830758, "PauseNs":[], "NumGC":36969} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 21:06:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.30800, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498405090, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886638592, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198193030, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86673055744, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 21:06:32 PDT 2021 RSS = 1861632 2021-09-13T21:06:34.649-07:00 [Info] memstats {"Alloc":64254248, "TotalAlloc":1855065683040, "Sys":283449592, "Lookups":0, "Mallocs":159920740,"Frees":159588898, "HeapAlloc":64254248, "HeapSys":264011776, "HeapIdle":195149824, "HeapInuse":68861952,"HeapReleased":178249728, "HeapObjects":331842,"GCSys":9134080, "LastGC":1631592331373230945,"PauseTotalNs":67289830758, "PauseNs":[], "NumGC":36969} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:07:04.687-07:00 [Info] memstats {"Alloc":65883112, "TotalAlloc":1855067311904, "Sys":283449592, "Lookups":0, "Mallocs":159939530,"Frees":159588907, "HeapAlloc":65883112, "HeapSys":264011776, "HeapIdle":194600960, "HeapInuse":69410816,"HeapReleased":178249728, "HeapObjects":350623,"GCSys":9134080, "LastGC":1631592331373230945,"PauseTotalNs":67289830758, "PauseNs":[], "NumGC":36969} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:07:32 PDT 2021 RSS = 1861632 2021-09-13T21:07:34.713-07:00 [Info] memstats {"Alloc":61324328, "TotalAlloc":1855068947176, "Sys":283449592, "Lookups":0, "Mallocs":159958357,"Frees":159658437, "HeapAlloc":61324328, "HeapSys":264011776, "HeapIdle":196141056, "HeapInuse":67870720,"HeapReleased":178249728, "HeapObjects":299920,"GCSys":9134080, "LastGC":1631592451516462528,"PauseTotalNs":67290175686, "PauseNs":[344928], "NumGC":36970} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:08:04.740-07:00 [Info] memstats {"Alloc":62947568, "TotalAlloc":1855070570416, "Sys":283449592, "Lookups":0, "Mallocs":159977151,"Frees":159658446, "HeapAlloc":62947568, "HeapSys":264011776, "HeapIdle":195633152, "HeapInuse":68378624,"HeapReleased":178249728, "HeapObjects":318705,"GCSys":9134080, "LastGC":1631592451516462528,"PauseTotalNs":67290175686, "PauseNs":[], "NumGC":36970} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:08:32 PDT 2021 RSS = 1861632 2021-09-13T21:08:34.768-07:00 [Info] memstats {"Alloc":64571240, "TotalAlloc":1855072194088, "Sys":283449592, "Lookups":0, "Mallocs":159995983,"Frees":159658455, "HeapAlloc":64571240, "HeapSys":264011776, "HeapIdle":194961408, "HeapInuse":69050368,"HeapReleased":178249728, "HeapObjects":337528,"GCSys":9134080, "LastGC":1631592451516462528,"PauseTotalNs":67290175686, "PauseNs":[], "NumGC":36970} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:09:04.795-07:00 [Info] memstats {"Alloc":66189752, "TotalAlloc":1855073812600, "Sys":283449592, "Lookups":0, "Mallocs":160014769,"Frees":159658464, "HeapAlloc":66189752, "HeapSys":264011776, "HeapIdle":194306048, "HeapInuse":69705728,"HeapReleased":178249728, "HeapObjects":356305,"GCSys":9134080, "LastGC":1631592451516462528,"PauseTotalNs":67290175686, "PauseNs":[], "NumGC":36970} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:09:32 PDT 2021 RSS = 1861632 2021-09-13T21:09:34.823-07:00 [Info] memstats {"Alloc":61784360, "TotalAlloc":1855075420448, "Sys":283449592, "Lookups":0, "Mallocs":160033600,"Frees":159727797, "HeapAlloc":61784360, "HeapSys":264011776, "HeapIdle":195796992, "HeapInuse":68214784,"HeapReleased":178249728, "HeapObjects":305803,"GCSys":9134080, "LastGC":1631592571658364852,"PauseTotalNs":67290538536, "PauseNs":[362850], "NumGC":36971} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:10:04.849-07:00 [Info] memstats {"Alloc":63400224, "TotalAlloc":1855077036312, "Sys":283449592, "Lookups":0, "Mallocs":160052389,"Frees":159727806, "HeapAlloc":63400224, "HeapSys":264011776, "HeapIdle":195141632, "HeapInuse":68870144,"HeapReleased":178249728, "HeapObjects":324583,"GCSys":9134080, "LastGC":1631592571658364852,"PauseTotalNs":67290538536, "PauseNs":[], "NumGC":36971} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:10:32 PDT 2021 RSS = 1861632 2021-09-13T21:10:34.878-07:00 [Info] memstats {"Alloc":67054496, "TotalAlloc":1855080690584, "Sys":283449592, "Lookups":0, "Mallocs":160074741,"Frees":159728438, "HeapAlloc":67054496, "HeapSys":264011776, "HeapIdle":192258048, "HeapInuse":71753728,"HeapReleased":178249728, "HeapObjects":346303,"GCSys":9134080, "LastGC":1631592571658364852,"PauseTotalNs":67290538536, "PauseNs":[], "NumGC":36971} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:11:04.905-07:00 [Info] memstats {"Alloc":68656512, "TotalAlloc":1855082292600, "Sys":283449592, "Lookups":0, "Mallocs":160093534,"Frees":159728447, "HeapAlloc":68656512, "HeapSys":264011776, "HeapIdle":191627264, "HeapInuse":72384512,"HeapReleased":178249728, "HeapObjects":365087,"GCSys":9134080, "LastGC":1631592571658364852,"PauseTotalNs":67290538536, "PauseNs":[], "NumGC":36971} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 21:11:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.28234, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498405090, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886638592, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198193030, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86673055744, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 21:11:32 PDT 2021 RSS = 1861632 2021-09-13T21:11:34.932-07:00 [Info] memstats {"Alloc":62287376, "TotalAlloc":1855083971344, "Sys":283449592, "Lookups":0, "Mallocs":160112597,"Frees":159800882, "HeapAlloc":62287376, "HeapSys":264011776, "HeapIdle":195141632, "HeapInuse":68870144,"HeapReleased":178249728, "HeapObjects":311715,"GCSys":9134080, "LastGC":1631592691770706922,"PauseTotalNs":67291549747, "PauseNs":[1011211], "NumGC":36972} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 21:11:42 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 21:11:45 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 21:11:46 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 21:11:48 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T21:12:04.958-07:00 [Info] memstats {"Alloc":65832952, "TotalAlloc":1855087516920, "Sys":283449592, "Lookups":0, "Mallocs":160132843,"Frees":159800938, "HeapAlloc":65832952, "HeapSys":264011776, "HeapIdle":192708608, "HeapInuse":71303168,"HeapReleased":178249728, "HeapObjects":331905,"GCSys":9134080, "LastGC":1631592691770706922,"PauseTotalNs":67291549747, "PauseNs":[], "NumGC":36972} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:12:32 PDT 2021 RSS = 1861632 2021-09-13T21:12:34.984-07:00 [Info] memstats {"Alloc":67435872, "TotalAlloc":1855089119840, "Sys":283449592, "Lookups":0, "Mallocs":160151672,"Frees":159800947, "HeapAlloc":67435872, "HeapSys":264011776, "HeapIdle":192036864, "HeapInuse":71974912,"HeapReleased":178249728, "HeapObjects":350725,"GCSys":9134080, "LastGC":1631592691770706922,"PauseTotalNs":67291549747, "PauseNs":[], "NumGC":36972} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:13:05.011-07:00 [Info] memstats {"Alloc":69075904, "TotalAlloc":1855090759872, "Sys":283449592, "Lookups":0, "Mallocs":160170474,"Frees":159800956, "HeapAlloc":69075904, "HeapSys":264011776, "HeapIdle":191430656, "HeapInuse":72581120,"HeapReleased":178249728, "HeapObjects":369518,"GCSys":9134080, "LastGC":1631592691770706922,"PauseTotalNs":67291549747, "PauseNs":[], "NumGC":36972} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:13:32 PDT 2021 RSS = 1861632 2021-09-13T21:13:35.037-07:00 [Info] memstats {"Alloc":62732728, "TotalAlloc":1855092395992, "Sys":283449592, "Lookups":0, "Mallocs":160189341,"Frees":159871725, "HeapAlloc":62732728, "HeapSys":264011776, "HeapIdle":194756608, "HeapInuse":69255168,"HeapReleased":178249728, "HeapObjects":317616,"GCSys":9134080, "LastGC":1631592811929883249,"PauseTotalNs":67291927026, "PauseNs":[377279], "NumGC":36973} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:14:05.063-07:00 [Info] memstats {"Alloc":64390032, "TotalAlloc":1855094053296, "Sys":283449592, "Lookups":0, "Mallocs":160208169,"Frees":159871734, "HeapAlloc":64390032, "HeapSys":264011776, "HeapIdle":194052096, "HeapInuse":69959680,"HeapReleased":178249728, "HeapObjects":336435,"GCSys":9134080, "LastGC":1631592811929883249,"PauseTotalNs":67291927026, "PauseNs":[], "NumGC":36973} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:14:32 PDT 2021 RSS = 1861632 2021-09-13T21:14:35.091-07:00 [Info] memstats {"Alloc":66036648, "TotalAlloc":1855095699912, "Sys":283449592, "Lookups":0, "Mallocs":160227030,"Frees":159871743, "HeapAlloc":66036648, "HeapSys":264011776, "HeapIdle":193404928, "HeapInuse":70606848,"HeapReleased":178249728, "HeapObjects":355287,"GCSys":9134080, "LastGC":1631592811929883249,"PauseTotalNs":67291927026, "PauseNs":[], "NumGC":36973} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:15:05.123-07:00 [Info] memstats {"Alloc":67678712, "TotalAlloc":1855097341976, "Sys":283449592, "Lookups":0, "Mallocs":160245847,"Frees":159871752, "HeapAlloc":67678712, "HeapSys":264011776, "HeapIdle":192724992, "HeapInuse":71286784,"HeapReleased":178249728, "HeapObjects":374095,"GCSys":9134080, "LastGC":1631592811929883249,"PauseTotalNs":67291927026, "PauseNs":[], "NumGC":36973} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:15:32 PDT 2021 RSS = 1861632 2021-09-13T21:15:35.149-07:00 [Info] memstats {"Alloc":63207832, "TotalAlloc":1855098984480, "Sys":283449592, "Lookups":0, "Mallocs":160264707,"Frees":159941203, "HeapAlloc":63207832, "HeapSys":264011776, "HeapIdle":194330624, "HeapInuse":69681152,"HeapReleased":178249728, "HeapObjects":323504,"GCSys":9134080, "LastGC":1631592932088319439,"PauseTotalNs":67292267229, "PauseNs":[340203], "NumGC":36974} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:16:05.175-07:00 [Info] memstats {"Alloc":64849088, "TotalAlloc":1855100625736, "Sys":283449592, "Lookups":0, "Mallocs":160283513,"Frees":159941212, "HeapAlloc":64849088, "HeapSys":264011776, "HeapIdle":193724416, "HeapInuse":70287360,"HeapReleased":178249728, "HeapObjects":342301,"GCSys":9134080, "LastGC":1631592932088319439,"PauseTotalNs":67292267229, "PauseNs":[], "NumGC":36974} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 21:16:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.37000, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498462434, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886695936, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198221702, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86673084416, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 21:16:33 PDT 2021 RSS = 1861632 2021-09-13T21:16:35.200-07:00 [Info] memstats {"Alloc":66571784, "TotalAlloc":1855102348432, "Sys":283449592, "Lookups":0, "Mallocs":160302622,"Frees":159941259, "HeapAlloc":66571784, "HeapSys":264011776, "HeapIdle":192872448, "HeapInuse":71139328,"HeapReleased":178249728, "HeapObjects":361363,"GCSys":9134080, "LastGC":1631592932088319439,"PauseTotalNs":67292267229, "PauseNs":[], "NumGC":36974} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:17:05.226-07:00 [Info] memstats {"Alloc":68222184, "TotalAlloc":1855103998832, "Sys":283449592, "Lookups":0, "Mallocs":160321435,"Frees":159941268, "HeapAlloc":68222184, "HeapSys":264011776, "HeapIdle":192184320, "HeapInuse":71827456,"HeapReleased":178249728, "HeapObjects":380167,"GCSys":9134080, "LastGC":1631592932088319439,"PauseTotalNs":67292267229, "PauseNs":[], "NumGC":36974} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:17:33 PDT 2021 RSS = 1861632 2021-09-13T21:17:35.252-07:00 [Info] memstats {"Alloc":63682328, "TotalAlloc":1855105653448, "Sys":283449592, "Lookups":0, "Mallocs":160340300,"Frees":160010924, "HeapAlloc":63682328, "HeapSys":264011776, "HeapIdle":193839104, "HeapInuse":70172672,"HeapReleased":178249728, "HeapObjects":329376,"GCSys":9138176, "LastGC":1631593052223323292,"PauseTotalNs":67292596509, "PauseNs":[329280], "NumGC":36975} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:18:05.279-07:00 [Info] memstats {"Alloc":65303536, "TotalAlloc":1855107274656, "Sys":283449592, "Lookups":0, "Mallocs":160359111,"Frees":160010933, "HeapAlloc":65303536, "HeapSys":264011776, "HeapIdle":193183744, "HeapInuse":70828032,"HeapReleased":178249728, "HeapObjects":348178,"GCSys":9138176, "LastGC":1631593052223323292,"PauseTotalNs":67292596509, "PauseNs":[], "NumGC":36975} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:18:33 PDT 2021 RSS = 1861632 2021-09-13T21:18:35.304-07:00 [Info] memstats {"Alloc":66921096, "TotalAlloc":1855108892216, "Sys":283449592, "Lookups":0, "Mallocs":160377955,"Frees":160010942, "HeapAlloc":66921096, "HeapSys":264011776, "HeapIdle":192626688, "HeapInuse":71385088,"HeapReleased":178249728, "HeapObjects":367013,"GCSys":9138176, "LastGC":1631593052223323292,"PauseTotalNs":67292596509, "PauseNs":[], "NumGC":36975} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:19:05.335-07:00 [Info] memstats {"Alloc":68554104, "TotalAlloc":1855110525224, "Sys":283449592, "Lookups":0, "Mallocs":160396779,"Frees":160010951, "HeapAlloc":68554104, "HeapSys":264011776, "HeapIdle":191692800, "HeapInuse":72318976,"HeapReleased":178249728, "HeapObjects":385828,"GCSys":9138176, "LastGC":1631593052223323292,"PauseTotalNs":67292596509, "PauseNs":[], "NumGC":36975} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:19:33 PDT 2021 RSS = 1861632 2021-09-13T21:19:35.364-07:00 [Info] memstats {"Alloc":64145832, "TotalAlloc":1855112136656, "Sys":283449592, "Lookups":0, "Mallocs":160415610,"Frees":160080343, "HeapAlloc":64145832, "HeapSys":264011776, "HeapIdle":193298432, "HeapInuse":70713344,"HeapReleased":178249728, "HeapObjects":335267,"GCSys":9142272, "LastGC":1631593172405358099,"PauseTotalNs":67292942561, "PauseNs":[346052], "NumGC":36976} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:20:05.390-07:00 [Info] memstats {"Alloc":65782480, "TotalAlloc":1855113773304, "Sys":283449592, "Lookups":0, "Mallocs":160434418,"Frees":160080352, "HeapAlloc":65782480, "HeapSys":264011776, "HeapIdle":192675840, "HeapInuse":71335936,"HeapReleased":178249728, "HeapObjects":354066,"GCSys":9142272, "LastGC":1631593172405358099,"PauseTotalNs":67292942561, "PauseNs":[], "NumGC":36976} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:20:33 PDT 2021 RSS = 1861632 2021-09-13T21:20:35.420-07:00 [Info] memstats {"Alloc":69453968, "TotalAlloc":1855117444792, "Sys":283449592, "Lookups":0, "Mallocs":160456798,"Frees":160080986, "HeapAlloc":69453968, "HeapSys":264011776, "HeapIdle":189784064, "HeapInuse":74227712,"HeapReleased":178249728, "HeapObjects":375812,"GCSys":9142272, "LastGC":1631593172405358099,"PauseTotalNs":67292942561, "PauseNs":[], "NumGC":36976} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:21:05.446-07:00 [Info] memstats {"Alloc":71061216, "TotalAlloc":1855119052040, "Sys":283449592, "Lookups":0, "Mallocs":160475591,"Frees":160080995, "HeapAlloc":71061216, "HeapSys":264011776, "HeapIdle":189112320, "HeapInuse":74899456,"HeapReleased":178249728, "HeapObjects":394596,"GCSys":9142272, "LastGC":1631593172405358099,"PauseTotalNs":67292942561, "PauseNs":[], "NumGC":36976} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 21:21:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.36800, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498462434, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886695936, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198221702, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86673084416, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 21:21:33 PDT 2021 RSS = 1861632 2021-09-13T21:21:35.473-07:00 [Info] memstats {"Alloc":64641472, "TotalAlloc":1855120714352, "Sys":283449592, "Lookups":0, "Mallocs":160494656,"Frees":160153508, "HeapAlloc":64641472, "HeapSys":264011776, "HeapIdle":192757760, "HeapInuse":71254016,"HeapReleased":178225152, "HeapObjects":341148,"GCSys":9142272, "LastGC":1631593292558048043,"PauseTotalNs":67293329122, "PauseNs":[386561], "NumGC":36977} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 21:21:42 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 21:21:45 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 21:21:46 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 21:21:48 [gsi-test] [1] CreateRecoveryPoint() - DO!!! 2021-09-13T21:22:05.499-07:00 [Info] memstats {"Alloc":68177032, "TotalAlloc":1855124249912, "Sys":283449592, "Lookups":0, "Mallocs":160514890,"Frees":160153564, "HeapAlloc":68177032, "HeapSys":264011776, "HeapIdle":190324736, "HeapInuse":73687040,"HeapReleased":178225152, "HeapObjects":361326,"GCSys":9142272, "LastGC":1631593292558048043,"PauseTotalNs":67293329122, "PauseNs":[], "NumGC":36977} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:22:33 PDT 2021 RSS = 1861632 2021-09-13T21:22:35.524-07:00 [Info] memstats {"Alloc":69757952, "TotalAlloc":1855125830832, "Sys":283449592, "Lookups":0, "Mallocs":160533715,"Frees":160153573, "HeapAlloc":69757952, "HeapSys":264011776, "HeapIdle":189743104, "HeapInuse":74268672,"HeapReleased":178225152, "HeapObjects":380142,"GCSys":9142272, "LastGC":1631593292558048043,"PauseTotalNs":67293329122, "PauseNs":[], "NumGC":36977} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:23:05.551-07:00 [Info] memstats {"Alloc":71331712, "TotalAlloc":1855127404592, "Sys":283449592, "Lookups":0, "Mallocs":160552504,"Frees":160153582, "HeapAlloc":71331712, "HeapSys":264011776, "HeapIdle":189030400, "HeapInuse":74981376,"HeapReleased":178225152, "HeapObjects":398922,"GCSys":9142272, "LastGC":1631593292558048043,"PauseTotalNs":67293329122, "PauseNs":[], "NumGC":36977} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:23:33 PDT 2021 RSS = 1861632 2021-09-13T21:23:35.575-07:00 [Info] memstats {"Alloc":65080808, "TotalAlloc":1855128979880, "Sys":283449592, "Lookups":0, "Mallocs":160571317,"Frees":160224326, "HeapAlloc":65080808, "HeapSys":264011776, "HeapIdle":192339968, "HeapInuse":71671808,"HeapReleased":178225152, "HeapObjects":346991,"GCSys":9142272, "LastGC":1631593412723480038,"PauseTotalNs":67293617052, "PauseNs":[287930], "NumGC":36978} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:24:05.601-07:00 [Info] memstats {"Alloc":66658864, "TotalAlloc":1855130557936, "Sys":283449592, "Lookups":0, "Mallocs":160590108,"Frees":160224335, "HeapAlloc":66658864, "HeapSys":264011776, "HeapIdle":191791104, "HeapInuse":72220672,"HeapReleased":178225152, "HeapObjects":365773,"GCSys":9142272, "LastGC":1631593412723480038,"PauseTotalNs":67293617052, "PauseNs":[], "NumGC":36978} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:24:33 PDT 2021 RSS = 1861632 2021-09-13T21:24:35.626-07:00 [Info] memstats {"Alloc":68212024, "TotalAlloc":1855132111096, "Sys":283449592, "Lookups":0, "Mallocs":160608931,"Frees":160224344, "HeapAlloc":68212024, "HeapSys":264011776, "HeapIdle":191324160, "HeapInuse":72687616,"HeapReleased":178225152, "HeapObjects":384587,"GCSys":9142272, "LastGC":1631593412723480038,"PauseTotalNs":67293617052, "PauseNs":[], "NumGC":36978} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:25:05.652-07:00 [Info] memstats {"Alloc":69777624, "TotalAlloc":1855133676696, "Sys":283449592, "Lookups":0, "Mallocs":160627709,"Frees":160224353, "HeapAlloc":69777624, "HeapSys":264011776, "HeapIdle":190734336, "HeapInuse":73277440,"HeapReleased":178225152, "HeapObjects":403356,"GCSys":9142272, "LastGC":1631593412723480038,"PauseTotalNs":67293617052, "PauseNs":[], "NumGC":36978} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:25:33 PDT 2021 RSS = 1861632 2021-09-13T21:25:35.681-07:00 [Info] memstats {"Alloc":65550856, "TotalAlloc":1855135260400, "Sys":283449592, "Lookups":0, "Mallocs":160646540,"Frees":160293670, "HeapAlloc":65550856, "HeapSys":264011776, "HeapIdle":191946752, "HeapInuse":72065024,"HeapReleased":178225152, "HeapObjects":352870,"GCSys":9142272, "LastGC":1631593532864464981,"PauseTotalNs":67294023927, "PauseNs":[406875], "NumGC":36979} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:26:05.710-07:00 [Info] memstats {"Alloc":67140832, "TotalAlloc":1855136850376, "Sys":283449592, "Lookups":0, "Mallocs":160665335,"Frees":160293679, "HeapAlloc":67140832, "HeapSys":264011776, "HeapIdle":191438848, "HeapInuse":72572928,"HeapReleased":178225152, "HeapObjects":371656,"GCSys":9142272, "LastGC":1631593532864464981,"PauseTotalNs":67294023927, "PauseNs":[], "NumGC":36979} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 21:26:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.33200, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498527970, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886761472, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198254470, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86673117184, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 21:26:33 PDT 2021 RSS = 1861632 2021-09-13T21:26:35.741-07:00 [Info] memstats {"Alloc":68812120, "TotalAlloc":1855138521664, "Sys":283449592, "Lookups":0, "Mallocs":160684397,"Frees":160293726, "HeapAlloc":68812120, "HeapSys":264011776, "HeapIdle":190636032, "HeapInuse":73375744,"HeapReleased":178225152, "HeapObjects":390671,"GCSys":9142272, "LastGC":1631593532864464981,"PauseTotalNs":67294023927, "PauseNs":[], "NumGC":36979} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:27:05.767-07:00 [Info] memstats {"Alloc":70398792, "TotalAlloc":1855140108336, "Sys":283449592, "Lookups":0, "Mallocs":160703195,"Frees":160293735, "HeapAlloc":70398792, "HeapSys":264011776, "HeapIdle":189947904, "HeapInuse":74063872,"HeapReleased":178225152, "HeapObjects":409460,"GCSys":9142272, "LastGC":1631593532864464981,"PauseTotalNs":67294023927, "PauseNs":[], "NumGC":36979} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:27:33 PDT 2021 RSS = 1861632 2021-09-13T21:27:35.800-07:00 [Info] memstats {"Alloc":66037016, "TotalAlloc":1855141724952, "Sys":283449592, "Lookups":0, "Mallocs":160722028,"Frees":160363241, "HeapAlloc":66037016, "HeapSys":264011776, "HeapIdle":191422464, "HeapInuse":72589312,"HeapReleased":178225152, "HeapObjects":358787,"GCSys":9142272, "LastGC":1631593653041022075,"PauseTotalNs":67294329580, "PauseNs":[305653], "NumGC":36980} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:28:05.824-07:00 [Info] memstats {"Alloc":67668192, "TotalAlloc":1855143356128, "Sys":283449592, "Lookups":0, "Mallocs":160740826,"Frees":160363250, "HeapAlloc":67668192, "HeapSys":264011776, "HeapIdle":190914560, "HeapInuse":73097216,"HeapReleased":178225152, "HeapObjects":377576,"GCSys":9142272, "LastGC":1631593653041022075,"PauseTotalNs":67294329580, "PauseNs":[], "NumGC":36980} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:28:33 PDT 2021 RSS = 1861632 2021-09-13T21:28:35.849-07:00 [Info] memstats {"Alloc":69286024, "TotalAlloc":1855144973960, "Sys":283449592, "Lookups":0, "Mallocs":160759668,"Frees":160363259, "HeapAlloc":69286024, "HeapSys":264011776, "HeapIdle":190177280, "HeapInuse":73834496,"HeapReleased":178225152, "HeapObjects":396409,"GCSys":9142272, "LastGC":1631593653041022075,"PauseTotalNs":67294329580, "PauseNs":[], "NumGC":36980} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:29:05.878-07:00 [Info] memstats {"Alloc":70891448, "TotalAlloc":1855146579384, "Sys":283449592, "Lookups":0, "Mallocs":160778456,"Frees":160363268, "HeapAlloc":70891448, "HeapSys":264011776, "HeapIdle":189505536, "HeapInuse":74506240,"HeapReleased":178225152, "HeapObjects":415188,"GCSys":9142272, "LastGC":1631593653041022075,"PauseTotalNs":67294329580, "PauseNs":[], "NumGC":36980} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:29:33 PDT 2021 RSS = 1861632 2021-09-13T21:29:35.904-07:00 [Info] memstats {"Alloc":66496568, "TotalAlloc":1855148176672, "Sys":283449592, "Lookups":0, "Mallocs":160797285,"Frees":160432634, "HeapAlloc":66496568, "HeapSys":264011776, "HeapIdle":191021056, "HeapInuse":72990720,"HeapReleased":178225152, "HeapObjects":364651,"GCSys":9142272, "LastGC":1631593773199315571,"PauseTotalNs":67294759223, "PauseNs":[429643], "NumGC":36981} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:30:05.936-07:00 [Info] memstats {"Alloc":68099424, "TotalAlloc":1855149779528, "Sys":283449592, "Lookups":0, "Mallocs":160816072,"Frees":160432643, "HeapAlloc":68099424, "HeapSys":264011776, "HeapIdle":190472192, "HeapInuse":73539584,"HeapReleased":178225152, "HeapObjects":383429,"GCSys":9142272, "LastGC":1631593773199315571,"PauseTotalNs":67294759223, "PauseNs":[], "NumGC":36981} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 Mon Sep 13 21:30:33 PDT 2021 RSS = 1861632 2021-09-13T21:30:35.967-07:00 [Info] memstats {"Alloc":71739200, "TotalAlloc":1855153419304, "Sys":283449592, "Lookups":0, "Mallocs":160838430,"Frees":160433277, "HeapAlloc":71739200, "HeapSys":264011776, "HeapIdle":187604992, "HeapInuse":76406784,"HeapReleased":178225152, "HeapObjects":405153,"GCSys":9142272, "LastGC":1631593773199315571,"PauseTotalNs":67294759223, "PauseNs":[], "NumGC":36981} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13T21:31:05.993-07:00 [Info] memstats {"Alloc":73365808, "TotalAlloc":1855155045912, "Sys":283449592, "Lookups":0, "Mallocs":160857216,"Frees":160433286, "HeapAlloc":73365808, "HeapSys":264011776, "HeapIdle":186908672, "HeapInuse":77103104,"HeapReleased":178225152, "HeapObjects":423930,"GCSys":9142272, "LastGC":1631593773199315571,"PauseTotalNs":67294759223, "PauseNs":[], "NumGC":36981} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 21:31:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.25200, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498527970, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886761472, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198254470, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86673117184, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 }Mon Sep 13 21:31:33 PDT 2021 RSS = 1861632 2021-09-13T21:31:36.024-07:00 [Info] memstats {"Alloc":67000496, "TotalAlloc":1855156744720, "Sys":283449592, "Lookups":0, "Mallocs":160876267,"Frees":160505706, "HeapAlloc":67000496, "HeapSys":264011776, "HeapIdle":190406656, "HeapInuse":73605120,"HeapReleased":178159616, "HeapObjects":370561,"GCSys":9142272, "LastGC":1631593893388560188,"PauseTotalNs":67295168123, "PauseNs":[408900], "NumGC":36982} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 21:31:42 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 21:31:46 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 21:31:46 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 21:31:49 [gsi-test] [1] CreateRecoveryPoint() - DO!!! GSI-Plasma Timer expired GSI-Plasma Time is over. Stop. 2021-09-13T21:31:51.045-07:00 [Info] memstats {"Alloc":69977160, "TotalAlloc":1855159721384, "Sys":283449592, "Lookups":0, "Mallocs":160887625,"Frees":160505762, "HeapAlloc":69977160, "HeapSys":264011776, "HeapIdle":188055552, "HeapInuse":75956224,"HeapReleased":178159616, "HeapObjects":381863,"GCSys":9142272, "LastGC":1631593893388560188,"PauseTotalNs":67295168123, "PauseNs":[], "NumGC":36982} Allocated: 1479928936, active: 1481420800, metadata: 22460528 (n_thp 0), resident: 1971806208, mapped: 1986408448, retained: 4051292160 2021-09-13 21:31:51 [gsi-test] [] MemoryStats... End Mon Sep 13 21:32:34 PDT 2021 RSS = 1861632 GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT Mon Sep 13 21:33:34 PDT 2021 RSS = 1861632 GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT Mon Sep 13 21:34:34 PDT 2021 RSS = 1861632 GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT Mon Sep 13 21:35:34 PDT 2021 RSS = 1861632 GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT 2021-09-13 21:36:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.26800, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498618082, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886851584, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198299526, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86673162240, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 } GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT Mon Sep 13 21:36:34 PDT 2021 RSS = 1861632 GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT Mon Sep 13 21:37:34 PDT 2021 RSS = 1861632 GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT Mon Sep 13 21:38:34 PDT 2021 RSS = 1861632 GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT Mon Sep 13 21:39:34 PDT 2021 RSS = 1861632 GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT Mon Sep 13 21:40:34 PDT 2021 RSS = 1861632 GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT 2021-09-13 21:41:27 [plasma] [i] Periodic Aggregated StorageStats: { "num_shard": 3, "num_instances": 10, "num_lssctx": 10, "num_sctxs": 267, "num_active_sctxs": 92, "num_free_sctxs": 175, "num_trim_sctxs": 19, "num_wctxs": 307, "num_free_wctxs": 245, "num_active_wctxs": 62, "num_trim_wctxs": 604, "num_readers": 103, "num_writers": 128, "cpu": 8.28600, "total_core": 16, "threads": 53, "assigned_quota": 994832636, "current_quota": 994832636, "memory_resident": 0, "memory_jemalloc": 1971806208, "memory_jemalloc_allocated": 1479928936, "memory_in_use": 1971806208, "memory_stats_size_total": 1244561288, "memory_stats_size_global": 895456604, "memory_stats_size_page": 759745140, "memory_stats_size_index": 135778072, "memory_stats_size_bloom": 0, "memory_stats_size_delta": 165368, "reclaim_pending": 0, "buf_memused": 349038076, "num_freeOS": 0, "num_smr_ctx ": 1, "num_smr_worker ": 32, "num_active_smr_worker ": 0, "num_total_smr": 10, "num_active_smr": 0, "num_pending_smr": 0, "num_pages": 2080, "items_count": 25, "total_records": 17986, "total_ops": 576589, "compacts": 2562003, "purges": 223, "lss_data_size": 713277490, "lss_used_space": 4498618082, "lss_fragmentation": 84, "lss_num_reads": 821767, "lss_read_bs": 130056343001, "lss_blk_read_bs": 126449172480, "rlss_num_reads": 32798, "lss_rdr_reads_bs": 11124928403, "lss_blk_rdr_reads_bs": 4088856576, "bytes_written": 116886851584, "bytes_incoming": 37794255772, "write_amp_avg": 1.00, "num_swapper_ctx": 3, "avg_sweep_interval": 146 s, "avg_run_interval": 98 ms, "avg_clock_interval": 4 s, "num_swapperWorker": 12, "num_swapperWriter": 80, "num_swapperPages": 0, "cached_records": 9387, "resident_ratio": 52.19059, "cleaner_num_reads": 788535, "cleaner_read_bs": 118105643100, "cleaner_blk_read_bs": 121617637376, "recovery_num_lssctx": 5, "recovery_num_sctxs": 39, "recovery_num_active_sctxs": 20, "recovery_num_free_sctxs": 19, "recovery_num_trim_sctxs": 19, "recovery_buf_memused": 5242880, "recovery_data_size": 177111012, "recovery_used_space": 1198299526, "recovery_lss_blk_read_bs": 89188888576, "recovery_lss_read_bs": 86648204942, "recovery_lss__num_reads": 667159, "recovery_bytes_written": 86673162240, "recovery_lss_frag": 85, "recovery_cleaner_num_reads": 667159, "recovery_cleaner_read_bs": 85960658019, "recovery_cleaner_blk_read_bs": 88502779904, "mvcc_purge_ratio": 719.44000 } GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT Mon Sep 13 21:41:34 PDT 2021 RSS = 1861632 GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT 2021-09-13 21:41:42 [gsi-test] [4] CreateRecoveryPoint() - DO!!! 2021-09-13 21:41:46 [gsi-test] [7] CreateRecoveryPoint() - DO!!! 2021-09-13 21:41:46 [gsi-test] [0] CreateRecoveryPoint() - DO!!! 2021-09-13 21:41:49 [gsi-test] [1] CreateRecoveryPoint() - DO!!! GSI-Plasma WAITING FOR ACTIVE INSTANCES TO EXIT goroutine 1550644 [running]: main.mainImpl.func1(0xc00015e250) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:645 +0xc0 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:635 +0xbf4 goroutine 1 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x2540be400) /usr/local/go/src/runtime/time.go:105 +0x157 main.mainImpl(0x7ffccccf727f, 0x20, 0xc000046101) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:657 +0xc87 main.main() /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:503 +0x69 goroutine 6 [chan receive, 2 minutes]: github.com/couchbase/plasma.runCleanerAutoTuner() /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/auto_tuner.go:651 +0x194 created by github.com/couchbase/plasma.init.2 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/shard.go:186 +0x18d goroutine 7 [chan receive, 2 minutes]: github.com/couchbase/plasma.singletonWorker() /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/shard.go:2389 +0xda created by github.com/couchbase/plasma.init.2 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/shard.go:187 +0x1a5 goroutine 8 [chan receive]: github.com/couchbase/plasma.systemResourceTracker() /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/mem.go:276 +0xcd created by github.com/couchbase/plasma.init.2 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/shard.go:188 +0x1bd goroutine 9 [chan receive, 2 minutes]: github.com/couchbase/plasma.AggregateAndLogStats() /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/shard.go:2355 +0x12f created by github.com/couchbase/plasma.init.2 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/shard.go:189 +0x1d5 goroutine 51 [chan receive]: github.com/couchbase/plasma.(*CleanerAutoTuner).refreshCleanerBandwidth(0xc000156100) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/auto_tuner.go:662 +0x9f created by github.com/couchbase/plasma.runCleanerAutoTuner /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/auto_tuner.go:648 +0x119 goroutine 66 [syscall, 190 minutes]: os/signal.signal_recv(0x0) /usr/local/go/src/runtime/sigqueue.go:147 +0x9c os/signal.loop() /usr/local/go/src/os/signal/signal_unix.go:23 +0x22 created by os/signal.init.0 /usr/local/go/src/os/signal/signal_unix.go:29 +0x41 goroutine 19 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x2540be400) /usr/local/go/src/runtime/time.go:105 +0x157 github.com/couchbase/indexing/secondary/common.pollForDeletedBuckets() /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/indexing/secondary/common/dcp_seqno.go:1060 +0x34 created by github.com/couchbase/indexing/secondary/common.init.0 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/indexing/secondary/common/dcp_seqno.go:52 +0x95 goroutine 20 [IO wait, 2 minutes]: internal/poll.runtime_pollWait(0x7fa100e0b108, 0x72, 0x0) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(*pollDesc).wait(0xc00004e198, 0x72, 0x0, 0x0, 0xc50b16) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Accept(0xc00004e180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1f8 net.(*netFD).accept(0xc00004e180, 0xc00008fd88, 0x701004, 0xc0000ec340) /usr/local/go/src/net/fd_unix.go:238 +0x42 net.(*TCPListener).accept(0xc00000eb40, 0x614027ad, 0xc00008fd88, 0x4c7516) /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 net.(*TCPListener).Accept(0xc00000eb40, 0xc00008fdd8, 0x18, 0xc0003b7500, 0x70051e) /usr/local/go/src/net/tcpsock.go:261 +0x47 net/http.(*Server).Serve(0xc0000ec2a0, 0xd5a100, 0xc00000eb40, 0x0, 0x0) /usr/local/go/src/net/http/server.go:2896 +0x286 net/http.(*Server).ListenAndServe(0xc0000ec2a0, 0xc0000ec2a0, 0x0) /usr/local/go/src/net/http/server.go:2825 +0xb7 net/http.ListenAndServe(0xc560f0, 0xe, 0x0, 0x0, 0xc00040e000, 0x7fa100e4cb10) /usr/local/go/src/net/http/server.go:3080 +0x74 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:512 +0x94 goroutine 84 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 85 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 86 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 87 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 88 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 89 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 90 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 91 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 92 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 93 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 94 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 95 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 96 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 97 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 98 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 99 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 100 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 101 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 102 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 103 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 104 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 105 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 106 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 107 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 108 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 109 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 110 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 111 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 112 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 113 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 114 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 115 [chan receive, 130 minutes]: github.com/couchbase/plasma.(*deallocCtx).smrWorker(0xc000192340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:344 +0xea created by github.com/couchbase/plasma.(*deallocCtx).startSMRWorker /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/smr.go:336 +0xc5 goroutine 121 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).mvccPurgerDaemon(0xc00001a000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/mvcc_purger.go:115 +0x218 created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:348 +0x4af goroutine 122 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).monitor(0xc00001a000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:434 +0x2bd created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:351 +0x3fb goroutine 128 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).mvccPurgerDaemon(0xc00001a600) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/mvcc_purger.go:115 +0x218 created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:348 +0x4af goroutine 129 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).monitor(0xc00001a600) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:434 +0x2bd created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:351 +0x3fb goroutine 188 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).mvccPurgerDaemon(0xc00001b800) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/mvcc_purger.go:115 +0x218 created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:348 +0x4af goroutine 189 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).monitor(0xc00001b800) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:434 +0x2bd created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:351 +0x3fb goroutine 190 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).mvccPurgerDaemon(0xc001174000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/mvcc_purger.go:115 +0x218 created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:348 +0x4af goroutine 191 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).monitor(0xc001174000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:434 +0x2bd created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:351 +0x3fb goroutine 193 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).mvccPurgerDaemon(0xc001692000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/mvcc_purger.go:115 +0x218 created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:348 +0x4af goroutine 194 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).monitor(0xc001692000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:434 +0x2bd created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:351 +0x3fb goroutine 195 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).mvccPurgerDaemon(0xc001174c00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/mvcc_purger.go:115 +0x218 created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:348 +0x4af goroutine 196 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).monitor(0xc001174c00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:434 +0x2bd created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:351 +0x3fb goroutine 197 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).mvccPurgerDaemon(0xc001174600) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/mvcc_purger.go:115 +0x218 created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:348 +0x4af goroutine 198 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).monitor(0xc001174600) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:434 +0x2bd created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:351 +0x3fb goroutine 157 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).mvccPurgerDaemon(0xc001693800) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/mvcc_purger.go:115 +0x218 created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:348 +0x4af goroutine 158 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).monitor(0xc001693800) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:434 +0x2bd created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:351 +0x3fb goroutine 159 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).mvccPurgerDaemon(0xc00001b200) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/mvcc_purger.go:115 +0x218 created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:348 +0x4af goroutine 160 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).monitor(0xc00001b200) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:434 +0x2bd created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:351 +0x3fb goroutine 161 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).mvccPurgerDaemon(0xc001175200) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/mvcc_purger.go:115 +0x218 created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:348 +0x4af goroutine 210 [select, 174 minutes]: github.com/couchbase/plasma.(*Plasma).monitor(0xc001175200) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:434 +0x2bd created by github.com/couchbase/plasma.New3 /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:351 +0x3fb goroutine 211 [semacquire, 70 minutes]: sync.runtime_Semacquire(0xc0003b94d4) /usr/local/go/src/runtime/sema.go:56 +0x42 sync.(*WaitGroup).Wait(0xc0003b94d4) /usr/local/go/src/sync/waitgroup.go:130 +0x64 main.(*InstanceCtx).StopWorkers(...) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:216 main.runIndexInstance(0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:262 +0x1c7 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:571 +0xa46 goroutine 212 [semacquire, 70 minutes]: sync.runtime_Semacquire(0xc0003b9714) /usr/local/go/src/runtime/sema.go:56 +0x42 sync.(*WaitGroup).Wait(0xc0003b9714) /usr/local/go/src/sync/waitgroup.go:130 +0x64 main.(*InstanceCtx).StopWorkers(...) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:216 main.runIndexInstance(0x1) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:262 +0x1c7 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:571 +0xa46 goroutine 213 [semacquire, 70 minutes]: sync.runtime_Semacquire(0xc0000fc2d4) /usr/local/go/src/runtime/sema.go:56 +0x42 sync.(*WaitGroup).Wait(0xc0000fc2d4) /usr/local/go/src/sync/waitgroup.go:130 +0x64 main.(*InstanceCtx).StopWorkers(...) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:216 main.runIndexInstance(0x2) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:262 +0x1c7 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:571 +0xa46 goroutine 214 [semacquire, 70 minutes]: sync.runtime_Semacquire(0xc0003b9dd4) /usr/local/go/src/runtime/sema.go:56 +0x42 sync.(*WaitGroup).Wait(0xc0003b9dd4) /usr/local/go/src/sync/waitgroup.go:130 +0x64 main.(*InstanceCtx).StopWorkers(...) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:216 main.runIndexInstance(0x3) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:262 +0x1c7 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:571 +0xa46 goroutine 215 [semacquire, 70 minutes]: sync.runtime_Semacquire(0xc0000fc094) /usr/local/go/src/runtime/sema.go:56 +0x42 sync.(*WaitGroup).Wait(0xc0000fc094) /usr/local/go/src/sync/waitgroup.go:130 +0x64 main.(*InstanceCtx).StopWorkers(...) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:216 main.runIndexInstance(0x4) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:262 +0x1c7 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:571 +0xa46 goroutine 216 [semacquire, 70 minutes]: sync.runtime_Semacquire(0xc001179dd4) /usr/local/go/src/runtime/sema.go:56 +0x42 sync.(*WaitGroup).Wait(0xc001179dd4) /usr/local/go/src/sync/waitgroup.go:130 +0x64 main.(*InstanceCtx).StopWorkers(...) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:216 main.runIndexInstance(0x5) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:262 +0x1c7 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:571 +0xa46 goroutine 217 [semacquire, 70 minutes]: sync.runtime_Semacquire(0xc001766094) /usr/local/go/src/runtime/sema.go:56 +0x42 sync.(*WaitGroup).Wait(0xc001766094) /usr/local/go/src/sync/waitgroup.go:130 +0x64 main.(*InstanceCtx).StopWorkers(...) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:216 main.runIndexInstance(0x6) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:262 +0x1c7 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:571 +0xa46 goroutine 218 [semacquire, 70 minutes]: sync.runtime_Semacquire(0xc0017662d4) /usr/local/go/src/runtime/sema.go:56 +0x42 sync.(*WaitGroup).Wait(0xc0017662d4) /usr/local/go/src/sync/waitgroup.go:130 +0x64 main.(*InstanceCtx).StopWorkers(...) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:216 main.runIndexInstance(0x7) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:262 +0x1c7 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:571 +0xa46 goroutine 219 [semacquire, 70 minutes]: sync.runtime_Semacquire(0xc001c7a094) /usr/local/go/src/runtime/sema.go:56 +0x42 sync.(*WaitGroup).Wait(0xc001c7a094) /usr/local/go/src/sync/waitgroup.go:130 +0x64 main.(*InstanceCtx).StopWorkers(...) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:216 main.runIndexInstance(0x8) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:262 +0x1c7 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:571 +0xa46 goroutine 220 [semacquire, 70 minutes]: sync.runtime_Semacquire(0xc001c7a2d4) /usr/local/go/src/runtime/sema.go:56 +0x42 sync.(*WaitGroup).Wait(0xc001c7a2d4) /usr/local/go/src/sync/waitgroup.go:130 +0x64 main.(*InstanceCtx).StopWorkers(...) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:216 main.runIndexInstance(0x9) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:262 +0x1c7 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:571 +0xa46 goroutine 221 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc0004480e0, 0xc00249d700, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).Lock(0xc0004480d8) /usr/local/go/src/sync/rwmutex.go:103 +0x88 main.(*InstanceCtx).createSnapshot(0xc001c7a240, 0xffffffffffffff01, 0xc00235a050) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:73 +0x74 main.(*InstanceCtx).CreateSnapshot(0xc001c7a240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:33 +0xf5 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:186 +0x82 goroutine 222 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x1312d00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).DeleteSnapshot(0xc001c7a240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:48 +0xe6 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:192 +0xf1 goroutine 223 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001d81200) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c03e80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doReadLast(0xc001c11800) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:227 +0xd9 main.(*WorkerCtx).Run(0xc001c11800, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:314 +0xb64 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 224 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc0004480e4, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc001c118c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 225 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc0016e2000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c03f00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc001c11980) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc001c11980, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 226 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc0004480e4, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc001c11a40, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 227 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc0004480e4, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc001c11b00, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 228 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc0004480e4, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc001c11bc0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 229 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc0004480e4, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc001c11c80, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:313 +0xbe7 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 230 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc0004480e4, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc001c11d40, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 231 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc0004480e4, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc001c11e00, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 232 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc0004480e4, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc001c11ec0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 233 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc0004480e4, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc001d36000, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 35 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x1312d00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).CreateSnapshot(0xc0000fc000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:35 +0xbb created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:186 +0x82 goroutine 234 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc000b459e0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001d16180) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doReadLast(0xc001d360c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:227 +0xd9 main.(*WorkerCtx).Run(0xc001d360c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:314 +0xb64 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 235 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc000b45050) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001d161c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc001d36180) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc001d36180, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 236 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc000450630) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001d16200) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doReadLast(0xc001d36240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:227 +0xd9 main.(*WorkerCtx).Run(0xc001d36240, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:314 +0xb64 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 237 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc000b45e60) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001d16240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc001d36300) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc001d36300, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 238 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc000451290) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001d16280) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doReadLast(0xc001d363c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:227 +0xd9 main.(*WorkerCtx).Run(0xc001d363c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:314 +0xb64 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 163 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x1312d00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).CreateSnapshot(0xc0003b9440) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:35 +0xbb created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:186 +0x82 goroutine 239 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448080, 0xc001e6b700, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).Lock(0xc000448078) /usr/local/go/src/sync/rwmutex.go:103 +0x88 main.(*InstanceCtx).createSnapshot(0xc001179d40, 0xffffffffffffff01, 0xc001bf42d0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:73 +0x74 main.(*InstanceCtx).CreateSnapshot(0xc001179d40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:33 +0xf5 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:186 +0x82 goroutine 240 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x1312d00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).DeleteSnapshot(0xc001179d40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:48 +0xe6 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:192 +0xf1 goroutine 241 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448084, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e8840, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 242 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448084, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e8900, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 243 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc0011aa1b0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc0016bdc80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc0016e89c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc0016e89c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 164 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x1312d00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).DeleteSnapshot(0xc0003b9440) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:48 +0xe6 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:192 +0xf1 goroutine 36 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x1312d00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).DeleteSnapshot(0xc0000fc000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:48 +0xe6 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:192 +0xf1 goroutine 165 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ea180) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ea180, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 166 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ea240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ea240, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 167 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ea300) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ea300, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 168 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ea3c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ea3c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 169 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ea480) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ea480, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 170 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ea540) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ea540, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 171 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ea600) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ea600, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 172 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ea6c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ea6c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 173 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ea780) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ea780, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 174 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ea840) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ea840, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 37 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ebbc0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ebbc0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 38 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ebc80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ebc80, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 175 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ea900) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ea900, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 39 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ebd40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ebd40, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 176 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ea9c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ea9c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 40 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ebe00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ebe00, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 41 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000ebec0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000ebec0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 177 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000eaa80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000eaa80, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 42 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0016e8000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0016e8000, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 258 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000eab40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000eab40, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 259 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000eac00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000eac00, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 43 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0016e80c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0016e80c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 260 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0000eacc0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0000eacc0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 44 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0016e8180) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0016e8180, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 200 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448038, 0xc001de0f00, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).Lock(0xc000448030) /usr/local/go/src/sync/rwmutex.go:103 +0x88 main.(*InstanceCtx).createSnapshot(0xc0000fc240, 0xffffffffffffff01, 0xc000134e10) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:73 +0x74 main.(*InstanceCtx).CreateSnapshot(0xc0000fc240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:33 +0xf5 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:186 +0x82 goroutine 45 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0016e8240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0016e8240, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 46 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0016e8300) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0016e8300, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 47 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0016e83c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0016e83c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 48 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0016e8480) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0016e8480, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 244 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001d80240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc0016bdcc0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc0016e8a80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc0016e8a80, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 49 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0016e8540) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0016e8540, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 274 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0016e8600) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0016e8600, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 275 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0016e86c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0016e86c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 245 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448084, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e8b40, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 246 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001d802d0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc0016bdd80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc0016e8c00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc0016e8c00, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 276 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc0016e8780) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc0016e8780, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 247 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc0016e26c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc0016bddc0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc0016e8cc0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc0016e8cc0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 248 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448084, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e8d80, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 249 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448084, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e8e40, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 250 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc0016e2510) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc0016bde80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc0016e8f00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc0016e8f00, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 251 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448084, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e8fc0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 201 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x1312d00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).DeleteSnapshot(0xc0000fc240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:48 +0xe6 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:192 +0xf1 goroutine 252 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448084, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e9080, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 253 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448084, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e9140, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 254 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448084, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e9200, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 255 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc00122c1b0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc0016bdfc0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc0016e92c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc0016e92c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 256 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448084, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e9380, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 202 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc000b45440) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117e600) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc00044d380) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc00044d380, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 203 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001d80990) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117e640) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc00044d440) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc00044d440, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 204 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc00044803c, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc00044d500, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 205 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001d80090) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117e6c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc00044d5c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc00044d5c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 206 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001eee750) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117e700) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc00044d680) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc00044d680, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 207 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc00044803c, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc00044d740, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 208 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc00044803c, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc00044d800, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 209 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001eee090) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117e800) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc00044d8c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc00044d8c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 290 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc00044803c, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc00044d980, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 291 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001d80ab0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117e880) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc00044da40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc00044da40, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 292 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc00044803c, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc00044db00, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 293 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc000b450e0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117e900) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc00044dbc0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc00044dbc0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 294 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001bd1680) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117e940) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc00044dc80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc00044dc80, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 295 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001d805a0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117e980) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc00044dd40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc00044dd40, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 296 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001d813b0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117e9c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc00044de00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc00044de00, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 297 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001bd10e0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117ea00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc00044dec0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc00044dec0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 257 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448098, 0xc001da9f00, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).Lock(0xc000448090) /usr/local/go/src/sync/rwmutex.go:103 +0x88 main.(*InstanceCtx).createSnapshot(0xc001766000, 0xffffffffffffff01, 0xc0004540f0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:73 +0x74 main.(*InstanceCtx).CreateSnapshot(0xc001766000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:33 +0xf5 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:186 +0x82 goroutine 322 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x1312d00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).CreateSnapshot(0xc0003b9680) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:35 +0xbb created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:186 +0x82 goroutine 261 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x1312d00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).CreateSnapshot(0xc001766240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:35 +0xbb created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:186 +0x82 goroutine 371 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x1312d00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).DeleteSnapshot(0xc001766000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:48 +0xe6 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:192 +0xf1 goroutine 372 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc000b457a0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc0017622c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc0016e9440) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc0016e9440, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 373 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc00044809c, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e9500, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 374 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc0016e2870) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001762340) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc0016e95c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc0016e95c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 375 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc00044809c, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e9680, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 376 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc00044809c, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e9740, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 377 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001bd01b0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001762440) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc0016e9800) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc0016e9800, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 378 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc000b44090) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001762480) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc0016e98c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc0016e98c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 379 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc00044809c, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e9980, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 380 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc00044809c, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e9a40, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 381 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001eeeb40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001762540) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc0016e9b00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc0016e9b00, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 382 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc00044809c, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e9bc0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 383 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001d80120) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc0017625c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc0016e9c80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc0016e9c80, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 384 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001d801b0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001762600) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc0016e9d40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc0016e9d40, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 385 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc00044809c, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0016e9e00, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 402 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001bd0480) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001762680) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc0016e9ec0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc0016e9ec0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 403 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc00044809c, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0017a8000, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 404 [semacquire, 170 minutes]: sync.runtime_SemacquireMutex(0xc0004480dc, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0xc0004480d8) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc0004480d8) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).CreateRecoveryPoint(0xc001c7a240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/recovery_point.go:17 +0x19a created by main.(*InstanceCtx).StartRecoveryPoint /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:238 +0xaf goroutine 405 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x1312d00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).DeleteSnapshot(0xc001766240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:48 +0xe6 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:192 +0xf1 goroutine 307 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448050, 0xc001dac700, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).Lock(0xc000448048) /usr/local/go/src/sync/rwmutex.go:103 +0x88 main.(*InstanceCtx).createSnapshot(0xc0003b9d40, 0xffffffffffffff01, 0xc001ea84b0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:73 +0x74 main.(*InstanceCtx).CreateSnapshot(0xc0003b9d40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:33 +0xf5 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:186 +0x82 goroutine 308 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c791c, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0x15c7918) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0x15c7918) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).Rollback(0xc001c7a240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/rollback.go:16 +0x186 created by main.(*InstanceCtx).StartRollback /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:255 +0xb1 goroutine 435 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc0004480c8, 0xc002d20700, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).Lock(0xc0004480c0) /usr/local/go/src/sync/rwmutex.go:103 +0x88 main.(*InstanceCtx).createSnapshot(0xc001c7a000, 0xffffffffffffff01, 0xc0001346e0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:73 +0x74 main.(*InstanceCtx).CreateSnapshot(0xc001c7a000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:33 +0xf5 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:186 +0x82 goroutine 436 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x1312d00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).DeleteSnapshot(0xc001c7a000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:48 +0xe6 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:192 +0xf1 goroutine 437 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc0004480cc, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc001c10c00, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 438 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001bd19e0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c03800) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc001c10cc0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc001c10cc0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 439 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc000451ef0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c03840) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc001c10d80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc001c10d80, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 440 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001d81170) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c03880) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc001c10e40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc001c10e40, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 441 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc000b44750) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c038c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc001c10f00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc001c10f00, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 442 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc0004480cc, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc001c10fc0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 443 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc000b44870) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c03980) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc001c11080) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc001c11080, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 444 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc000b458c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c039c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc001c11140) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc001c11140, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 445 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc000b45dd0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c03a00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc001c11200) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc001c11200, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 446 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc0011aa240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c03a40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doReadLast(0xc001c112c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:227 +0xd9 main.(*WorkerCtx).Run(0xc001c112c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:314 +0xb64 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 447 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001bd02d0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c03a80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc001c11380) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc001c11380, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 448 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001bd1200) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c03ac0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doReadLast(0xc001c11440) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:227 +0xd9 main.(*WorkerCtx).Run(0xc001c11440, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:314 +0xb64 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 449 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001eee000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c03b00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc001c11500) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc001c11500, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 450 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc0011aa2d0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c03b40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doReadLast(0xc001c115c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:227 +0xd9 main.(*WorkerCtx).Run(0xc001c115c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:314 +0xb64 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 451 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001bd0a20) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c03b80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc001c11680) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc001c11680, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 452 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc0004503f0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc001c03bc0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc001c11740) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc001c11740, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 453 [semacquire, 170 minutes]: sync.runtime_SemacquireMutex(0xc0004480c4, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0xc0004480c0) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc0004480c0) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).CreateRecoveryPoint(0xc001c7a000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/recovery_point.go:17 +0x19a created by main.(*InstanceCtx).StartRecoveryPoint /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:238 +0xaf goroutine 454 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c791c, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0x15c7918) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0x15c7918) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).Rollback(0xc001c7a000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/rollback.go:16 +0x186 created by main.(*InstanceCtx).StartRollback /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:255 +0xb1 goroutine 455 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x3b9aca00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).Generation(0xc001c7a000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/generation.go:24 +0xf6 created by main.(*InstanceCtx).StartGeneration /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:270 +0x8b goroutine 131 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x1312d00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).DeleteSnapshot(0xc0003b9680) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:48 +0xe6 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:192 +0xf1 goroutine 132 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044c480) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044c480, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 133 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044c6c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044c6c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 134 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044c780) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044c780, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 135 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044c840) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044c840, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 136 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044c900) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044c900, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 137 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044c9c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044c9c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 138 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044ca80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044ca80, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 139 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044cb40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044cb40, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 140 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044cc00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044cc00, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 141 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044ccc0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044ccc0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 142 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044cd80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044cd80, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 143 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044ce40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044ce40, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 144 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044cf00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044cf00, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 145 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044cfc0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044cfc0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 482 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044d080) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044d080, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 483 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScan(0xc00044d140) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:244 +0x13a main.(*WorkerCtx).Run(0xc00044d140, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:318 +0xc31 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 407 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x3b9aca00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).CreateRecoveryPoint(0xc0003b9680) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/recovery_point.go:54 +0x105 created by main.(*InstanceCtx).StartRecoveryPoint /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:238 +0xaf goroutine 52 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x3b9aca00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).CreateRecoveryPoint(0xc0003b9440) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/recovery_point.go:54 +0x105 created by main.(*InstanceCtx).StartRecoveryPoint /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:238 +0xaf goroutine 53 [semacquire, 170 minutes]: sync.runtime_SemacquireMutex(0xc000448034, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0xc000448030) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc000448030) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).CreateRecoveryPoint(0xc0000fc240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/recovery_point.go:17 +0x19a created by main.(*InstanceCtx).StartRecoveryPoint /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:238 +0xaf goroutine 54 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c791c, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0x15c7918) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0x15c7918) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).Rollback(0xc0000fc240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/rollback.go:16 +0x186 created by main.(*InstanceCtx).StartRollback /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:255 +0xb1 goroutine 55 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x3b9aca00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).Generation(0xc0000fc240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/generation.go:24 +0xf6 created by main.(*InstanceCtx).StartGeneration /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:270 +0x8b goroutine 75 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c791c, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0x15c7918) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0x15c7918) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).Rollback(0xc0003b9680) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/rollback.go:16 +0x186 created by main.(*InstanceCtx).StartRollback /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:255 +0xb1 goroutine 76 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x3b9aca00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).Generation(0xc0003b9680) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/generation.go:24 +0xf6 created by main.(*InstanceCtx).StartGeneration /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:270 +0x8b goroutine 309 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x1312d00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).DeleteSnapshot(0xc0003b9d40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/snapshot.go:48 +0xe6 created by main.(*InstanceCtx).StartSnapshot /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:192 +0xf1 goroutine 310 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001bd0d80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117ed00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doRead(0xc0000eae40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:201 +0x10e main.(*WorkerCtx).Run(0xc0000eae40, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:310 +0xa7f created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 311 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448054, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0000eaf00, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 312 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448054, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0000eafc0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 313 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448054, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0000eb080, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 314 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448054, 0x0, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0000eb140, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:297 +0x853 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 315 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448054, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0000eb200, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 316 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc000b45d40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117eec0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc0000eb2c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc0000eb2c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 317 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc0016e2750) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117ef00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc0000eb380) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc0000eb380, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 318 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448054, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0000eb440, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 319 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448054, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0000eb500, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 320 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc001d81050) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117efc0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc0000eb5c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc0000eb5c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 321 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448054, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0000eb680, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 498 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc0016e2630) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117f040) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc0000eb740) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc0000eb740, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 499 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448054, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0000eb800, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 500 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448054, 0xc0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).Run(0xc0000eb8c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:326 +0x815 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 501 [runnable]: runtime.Gosched(...) /usr/local/go/src/runtime/proc.go:269 github.com/couchbase/plasma.(*wCtx).tryThrottleForLSS(0xc0016e3950) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/plasma.go:1593 +0x44 github.com/couchbase/plasma.(*Writer).Begin(0xc00117f100) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/writer.go:45 +0x84 main.(*WorkerCtx).doUpdate(0xc0000eb980) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:124 +0x2b9 main.(*WorkerCtx).Run(0xc0000eb980, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:302 +0x8b5 created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 484 [semacquire, 170 minutes]: sync.runtime_SemacquireMutex(0xc000448094, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0xc000448090) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc000448090) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).CreateRecoveryPoint(0xc001766000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/recovery_point.go:17 +0x19a created by main.(*InstanceCtx).StartRecoveryPoint /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:238 +0xaf goroutine 485 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c791c, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0x15c7918) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0x15c7918) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).Rollback(0xc001766000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/rollback.go:16 +0x186 created by main.(*InstanceCtx).StartRollback /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:255 +0xb1 goroutine 486 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x3b9aca00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).Generation(0xc001766000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/generation.go:24 +0xf6 created by main.(*InstanceCtx).StartGeneration /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:270 +0x8b goroutine 408 [semacquire, 170 minutes]: sync.runtime_SemacquireMutex(0xc00044804c, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0xc000448048) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc000448048) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).CreateRecoveryPoint(0xc0003b9d40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/recovery_point.go:17 +0x19a created by main.(*InstanceCtx).StartRecoveryPoint /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:238 +0xaf goroutine 409 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0xc00044804c, 0xc000456e00, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0xc000448048) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc000448048) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).Rollback(0xc0003b9d40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/rollback.go:21 +0x1c6 created by main.(*InstanceCtx).StartRollback /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:255 +0xb1 goroutine 410 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x3b9aca00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).Generation(0xc0003b9d40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/generation.go:24 +0xf6 created by main.(*InstanceCtx).StartGeneration /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:270 +0x8b goroutine 56 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x3b9aca00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).Generation(0xc001c7a240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/generation.go:24 +0xf6 created by main.(*InstanceCtx).StartGeneration /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:270 +0x8b goroutine 514 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c791c, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0x15c7918) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0x15c7918) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).Rollback(0xc0003b9440) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/rollback.go:16 +0x186 created by main.(*InstanceCtx).StartRollback /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:255 +0xb1 goroutine 515 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x3b9aca00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).Generation(0xc0003b9440) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/generation.go:24 +0xf6 created by main.(*InstanceCtx).StartGeneration /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:270 +0x8b goroutine 516 [semacquire, 170 minutes]: sync.runtime_SemacquireMutex(0xc00044807c, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0xc000448078) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc000448078) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).CreateRecoveryPoint(0xc001179d40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/recovery_point.go:17 +0x19a created by main.(*InstanceCtx).StartRecoveryPoint /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:238 +0xaf goroutine 517 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c791c, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0x15c7918) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0x15c7918) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).Rollback(0xc001179d40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/rollback.go:16 +0x186 created by main.(*InstanceCtx).StartRollback /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:255 +0xb1 goroutine 518 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x3b9aca00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).Generation(0xc001179d40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/generation.go:24 +0xf6 created by main.(*InstanceCtx).StartGeneration /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:270 +0x8b goroutine 57 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c10000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c10000, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 58 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c100c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c100c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 59 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c10180) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c10180, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 60 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c10240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c10240, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 61 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c10300) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c10300, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 62 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c103c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c103c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 63 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c10480) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c10480, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 64 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c10540) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c10540, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 65 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c10600) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c10600, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 530 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c106c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c106c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 531 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c10780) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c10780, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 532 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c10840) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c10840, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 533 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c10900) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c10900, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 534 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c109c0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c109c0, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 535 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c10a80) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c10a80, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 536 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c7924, 0x8000000000000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.(*WorkerCtx).doScanLast(0xc001c10b40) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:257 +0x13a main.(*WorkerCtx).Run(0xc001c10b40, 0x0) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/worker.go:322 +0xc1e created by main.(*InstanceCtx).StartWorkers /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:209 +0x81 goroutine 298 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x3b9aca00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).CreateRecoveryPoint(0xc001766240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/recovery_point.go:54 +0x105 created by main.(*InstanceCtx).StartRecoveryPoint /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:238 +0xaf goroutine 299 [semacquire, 70 minutes]: sync.runtime_SemacquireMutex(0x15c791c, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0x15c7918) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0x15c7918) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).Rollback(0xc001766240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/rollback.go:16 +0x186 created by main.(*InstanceCtx).StartRollback /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:255 +0xb1 goroutine 300 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x3b9aca00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).Generation(0xc001766240) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/generation.go:24 +0xf6 created by main.(*InstanceCtx).StartGeneration /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:270 +0x8b goroutine 411 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x3b9aca00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).CreateRecoveryPoint(0xc0000fc000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/recovery_point.go:54 +0x105 created by main.(*InstanceCtx).StartRecoveryPoint /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:238 +0xaf goroutine 412 [semacquire, 130 minutes]: sync.runtime_SemacquireMutex(0x15c791c, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0x15c7918) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0x15c7918) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.(*InstanceCtx).Rollback(0xc0000fc000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/rollback.go:16 +0x186 created by main.(*InstanceCtx).StartRollback /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:255 +0xb1 goroutine 413 [sleep]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 time.Sleep(0x3b9aca00) /usr/local/go/src/runtime/time.go:105 +0x157 main.(*InstanceCtx).Generation(0xc0000fc000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/generation.go:24 +0xf6 created by main.(*InstanceCtx).StartGeneration /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/instance.go:270 +0x8b goroutine 1592 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc00044803c, 0xc0c0000000, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.shortStats(0xdf8475800, 0x100) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:401 +0x853 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:584 +0xece goroutine 1593 [semacquire, 169 minutes]: sync.runtime_SemacquireMutex(0xc00044803c, 0xffffffffffffff00, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.fullStats(0x45d964b800, 0x101) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:323 +0x3d1 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:590 +0xe76 goroutine 1594 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448034, 0x471500, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0xc000448030) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc000448030) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.QuotaDecision(0x3b9aca00) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/quota_tuner.go:11 +0x9b main.QuotaTuner(0x140000000) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/quota_tuner.go:36 +0xd5 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:599 +0xe1e goroutine 1595 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc00044803c, 0xc0002db700, 0x0) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*RWMutex).RLock(...) /usr/local/go/src/sync/rwmutex.go:50 main.Verify() /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/verify.go:60 +0x439 created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:605 +0xda9 goroutine 1596 [semacquire, 174 minutes]: sync.runtime_SemacquireMutex(0xc000448034, 0xc00122e100, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0xc000448030) /usr/local/go/src/sync/mutex.go:138 +0xfc sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 sync.(*RWMutex).Lock(0xc000448030) /usr/local/go/src/sync/rwmutex.go:98 +0x97 main.StopAndVerify() /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/verify.go:103 +0x2bb created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:611 +0xd73 ---------- panic: Forced termination! Please, check the code! goroutine 1550644 [running]: main.mainImpl.func1(0xc00015e250) /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:648 +0x1cb created by main.mainImpl /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/gsi.go:635 +0xbf4 + grep -Ei panic /data/jenkins/workspace/centos-2i_plasma_API_tests/cheshire-cat/goproj/src/github.com/couchbase/plasma/tools/gsi/output.txt panic: Forced termination! Please, check the code! + exit 1 Process leaked file descriptors. See https://jenkins.io/redirect/troubleshooting/process-leaked-file-descriptors for more information Build step 'Execute shell' marked build as failure /data/jenkins/workspace/centos-2i_plasma_API_tests/heapprofile.sh: line 1: 94551 Terminated sleep 600 [description-setter] Description set: [BFA] Scanning build for known causes... [BFA] No failure causes found [BFA] Done. 1s Notifying upstream projects of job completion Email was triggered for: Failure - Any Sending email for trigger: Failure - Any Sending email to: girish.benakappa@couchbase.com dmitriy.kalugin-balashov@couchbase.com mihir.kamdar@couchbase.com Finished: FAILURE