[system test] High Swap ~ 10G on indexer node, 50% of this is from the indexer process

1. Load 10M items on the cluster
2. Indexes are built, steady state
3. Reboot cluster – the indexes appear to be ready (via getIndexStatus, system:ndexes etc)
4. Seeing the Swap space on the machine continusly increasing,

Used "smem" to print out which process is swapping - Indexer, which shows heavy swapping

[root@centos-64-x64 tmp]# pidof indexer
4606
[root@centos-64-x64 tmp]# grep --color VmSwap /proc/4606/status
VmSwap: 7609396 kB

[root@centos-64-x64 tmp]# cat /proc/meminfo | grep Swap
SwapCached: 298008 kB
SwapTotal: 10829816 kB
SwapFree: 1966096 kB

[root@centos-64-x64 tmp]# smem -t -p
PID User Command Swap USS PSS RSS
875 root /sbin/dhclient -1 -q -cf /e 0.01% 0.00% 0.00% 0.00%
401 root /sbin/udevd -d 0.00% 0.00% 0.00% 0.00%
610 root /sbin/udevd -d 0.00% 0.00% 0.00% 0.00%
607 root /sbin/udevd -d 0.00% 0.00% 0.00% 0.00%
1253 couchbase inet_gethost 4 0.00% 0.00% 0.00% 0.00%
1223 couchbase /opt/couchbase/lib/erlang/l 0.00% 0.00% 0.00% 0.00%
1288 couchbase /opt/couchbase/lib/erlang/l 0.00% 0.00% 0.00% 0.00%
1397 couchbase inet_gethost 4 0.00% 0.00% 0.00% 0.00%
1382 couchbase inet_gethost 4 0.00% 0.00% 0.00% 0.00%
1241 root /sbin/mingetty /dev/tty1 0.00% 0.00% 0.00% 0.00%
1243 root /sbin/mingetty /dev/tty2 0.00% 0.00% 0.00% 0.00%
1245 root /sbin/mingetty /dev/tty3 0.00% 0.00% 0.00% 0.00%
1247 root /sbin/mingetty /dev/tty4 0.00% 0.00% 0.00% 0.00%
1249 root /sbin/mingetty /dev/tty5 0.00% 0.00% 0.00% 0.00%
1251 root /sbin/mingetty /dev/tty6 0.00% 0.00% 0.00% 0.00%
1240 root /sbin/agetty /dev/hvc0 3840 0.00% 0.00% 0.00% 0.00%
1 root /sbin/init 0.00% 0.00% 0.00% 0.00%
1226 root /usr/bin/ruby /usr/bin/chef 0.29% 0.00% 0.00% 0.00%
1252 couchbase inet_gethost 4 0.00% 0.00% 0.00% 0.00%
1299 couchbase inet_gethost 4 0.00% 0.00% 0.00% 0.00%
1065 couchbase /opt/couchbase/lib/erlang/e 0.00% 0.00% 0.00% 0.00%
1287 couchbase /opt/couchbase/lib/erlang/l 0.00% 0.00% 0.00% 0.00%
1222 couchbase /opt/couchbase/lib/erlang/l 0.00% 0.00% 0.00% 0.00%
7032 root sleep 60 0.00% 0.00% 0.00% 0.00%
1141 root /usr/sbin/sshd 0.01% 0.00% 0.00% 0.00%
1192 root crond 0.00% 0.00% 0.00% 0.00%
1286 couchbase sh -s disksup 0.00% 0.00% 0.00% 0.00%
916 root auditd 0.00% 0.00% 0.00% 0.00%
1292 couchbase sh -s ns_disksup 0.00% 0.00% 0.00% 0.00%
932 root /sbin/rsyslogd -i /var/run/ 0.00% 0.00% 0.00% 0.00%
1301 couchbase portsigar for ns_1@10.6.2.1 0.00% 0.00% 0.00% 0.00%
1534 couchbase /opt/couchbase/bin/goport 0.00% 0.00% 0.00% 0.00%
1549 couchbase /opt/couchbase/bin/goport 0.00% 0.00% 0.00% 0.00%
1221 couchbase sh -s disksup 0.00% 0.00% 0.00% 0.00%
953 root /bin/bash /usr/sbin/xe-daem 0.00% 0.00% 0.00% 0.00%
1300 couchbase /opt/couchbase/bin/moxi -Z 0.08% 0.00% 0.00% 0.00%
1150 ntp ntpd -u ntp:ntp -p /var/run 0.00% 0.00% 0.00% 0.00%
1509 couchbase /opt/couchbase/bin/goport 0.00% 0.00% 0.00% 0.00%
1293 couchbase /opt/couchbase/bin/priv/god 0.00% 0.00% 0.00% 0.00%
4604 couchbase /opt/couchbase/bin/goport 0.00% 0.00% 0.00% 0.00%
5980 root -bash 0.00% 0.00% 0.00% 0.01%
6837 root -bash 0.00% 0.00% 0.00% 0.01%
5978 root sshd: root@pts/0 0.00% 0.00% 0.00% 0.01%
6835 root sshd: root@pts/1 0.00% 0.00% 0.00% 0.01%
1327 couchbase /opt/couchbase/bin/saslauth 0.00% 0.01% 0.01% 0.01%
1291 couchbase /opt/couchbase/bin/priv/god 0.00% 0.00% 0.01% 0.01%
1343 couchbase /opt/couchbase/lib/erlang/e 0.22% 0.01% 0.01% 0.01%
1551 couchbase /opt/couchbase/bin/goxdcr - 0.01% 0.02% 0.02% 0.02%
1101 couchbase /opt/couchbase/lib/erlang/e 0.26% 0.02% 0.03% 0.03%
7037 root python /usr/local/bin/smem 0.00% 0.03% 0.03% 0.03%
1167 root /usr/bin/atop -a -w /var/lo 0.00% 0.03% 0.03% 0.03%
1539 couchbase /opt/couchbase/bin/cbq-engi 0.11% 0.04% 0.04% 0.04%
1254 couchbase /opt/couchbase/lib/erlang/e 0.28% 0.17% 0.18% 0.18%
1511 couchbase /opt/couchbase/bin/projecto 0.19% 0.79% 0.79% 0.79%
1159 couchbase /opt/couchbase/lib/erlang/e 4.77% 2.76% 2.76% 2.77%
1380 couchbase /opt/couchbase/bin/memcache 24.17% 6.23% 6.23% 6.24%
4606 couchbase /opt/couchbase/bin/indexer 50.14% 85.78% 85.78% 85.78%
-------------------------------------------------------------------------------
57 3 80.57% 95.92% 95.93% 96.06%